From a60a49ade0cf6b1fc8309be870677db461972d75 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Mon, 2 Oct 2023 16:26:04 +0200 Subject: [PATCH 01/19] Implemented SQLSink --- .../ie3/datamodel/io/DatabaseIdentifier.java | 43 ++ .../java/edu/ie3/datamodel/io/IoUtil.java | 15 + .../java/edu/ie3/datamodel/io/SqlUtils.java | 192 +++++++ .../datamodel/io/connectors/SqlConnector.java | 16 +- .../io/naming/DatabaseNamingStrategy.java | 51 ++ .../ie3/datamodel/io/processor/Processor.java | 3 +- .../io/processor/ProcessorProvider.java | 12 + .../edu/ie3/datamodel/io/sink/SqlSink.java | 507 ++++++++++++++++++ .../datamodel/io/sink/CsvFileSinkTest.groovy | 2 +- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 402 ++++++++++++++ .../ie3/test/common/SampleJointGrid.groovy | 4 +- .../ie3/test/common/TimeSeriesTestData.groovy | 7 + .../ie3/datamodel/io/sink/_sql/cleanup.sql | 7 + .../datamodel/io/sink/_sql/input_entities.sql | 240 +++++++++ .../datamodel/io/sink/_sql/load_profile.sql | 13 + .../io/sink/_sql/result_entities.sql | 79 +++ .../edu/ie3/datamodel/io/sink/_sql/setup.sql | 1 + .../datamodel/io/sink/_sql/sql_test_data.sql | 16 + .../datamodel/io/sink/_sql/time_series.sql | 123 +++++ .../edu/ie3/datamodel/io/sink/_sql/types.sql | 59 ++ .../datamodel/io/source/sql/_types/types.sql | 2 +- src/test/resources/log4j2-test.xml | 2 +- 22 files changed, 1783 insertions(+), 13 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java create mode 100644 src/main/java/edu/ie3/datamodel/io/SqlUtils.java create mode 100644 src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java create mode 100644 src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/cleanup.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/setup.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql diff --git a/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java b/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java new file mode 100644 index 000000000..14d9c0738 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java @@ -0,0 +1,43 @@ +package edu.ie3.datamodel.io; + +import java.util.UUID; +import java.util.stream.Stream; + +import static edu.ie3.datamodel.io.IoUtil.quote; + +/** + * Class for identification of grids and results in SQL databases. + */ +public class DatabaseIdentifier { + + private final String identifier; + private final UUID uuid; + + public DatabaseIdentifier( + String identifier, + UUID uuid + ) { + this.identifier = identifier; + this.uuid = uuid; + } + + public String getIdentifier() { + return identifier; + } + + public UUID getUuid() { + return uuid; + } + + public String toString() { + return "identifier=" + identifier + ", uuid=" + uuid.toString(); + } + + public String[] getQueryString() { + return new String[]{quote(identifier, "'"), quote(uuid.toString(), "'")}; + } + + public Stream getStreamForQuery() { + return Stream.concat(Stream.of(quote(identifier, "'")), Stream.of(quote(uuid.toString(), "'"))); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/IoUtil.java b/src/main/java/edu/ie3/datamodel/io/IoUtil.java index 0968b4f28..590ea1b49 100644 --- a/src/main/java/edu/ie3/datamodel/io/IoUtil.java +++ b/src/main/java/edu/ie3/datamodel/io/IoUtil.java @@ -7,6 +7,8 @@ import java.io.File; import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Optional; public class IoUtil { @@ -49,4 +51,17 @@ public static Path harmonizeFileSeparator(Path path) { public static Optional pathOption(String in) { return Optional.of(Path.of(in)); } + + public String timeFormatter(ZonedDateTime time, String timePattern) { + return time.format(DateTimeFormatter.ofPattern(timePattern)); + } + + public static String quote(String input, String quoteSymbol) { + if (input == "") { + return "NULL"; + } else { + return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; + } + } + } diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java new file mode 100644 index 000000000..c0fe83645 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -0,0 +1,192 @@ +package edu.ie3.datamodel.io; + +import edu.ie3.datamodel.exceptions.EntityProcessorException; +import edu.ie3.datamodel.exceptions.ProcessorProviderException; +import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; +import edu.ie3.datamodel.io.processor.ProcessorProvider; +import edu.ie3.datamodel.models.UniqueEntity; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; + +public class SqlUtils { + private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; + + private SqlUtils() { + throw new IllegalStateException("Utility classes cannot be instantiated"); + } + + + public static String queryForCreation( + String schemaName, + String tableName, + Stream> columnsWithDataTypes + ) { + return beginQueryCreateTable(schemaName, tableName) + + " " + + endQueryCreateTable; + } + + public static String getEndQueryCreateTable() { + return endQueryCreateTable; + } + + private static String beginQueryCreateTable( + String schemaName, + String tableName + ) { + return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; + } + + public static String getDataTypes(Class cls) { + try { + ProcessorProvider processorProvider = new ProcessorProvider(); + DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); + String body = ""; + Stream dataTypes; + String[] headerElements = processorProvider.getHeaderElements(cls); + Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_name", "grid_uuid")) ; + Stream dtHeader = strHeader.map( + element -> { + return camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element)); + } + ); + return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" + + "WITHOUT OIDS\n" + + "\t" + "TABLESPACE pg_default;\n"; + } catch (EntityProcessorException e) { + return ""; + } catch (ProcessorProviderException e) { + return ""; + } + } + + public static Map classToDataType() { + HashMap map = new HashMap(); + map.put("uuid", "uuid PRIMARY KEY"); + map.put("time_series", "uuid NOT NULL"); + map.put("time", "timestamp with time zone NOT NULL"); + map.put("p", "double precision NOT NULL"); + map.put("q", "double precision NOT NULL"); + map.put("c", "double precision NOT NULL"); + map.put("s_rated", "double precision NOT NULL"); + + map.put("cost_controlled", "bool NOT NULL"); + map.put("feed_in_tariff", "int NOT NULL"); + map.put("id", "TEXT NOT NULL"); + map.put("market_reaction", "bool NOT NULL"); + map.put("node", "uuid NOT NULL"); + //map.put("operatesFrom", "timestamp with time zone"); + map.put("operates_from", "timestamp with time zone"); + //map.put("operatesUntil", "timestamp with time zone"); + map.put("operates_until", "timestamp with time zone"); + map.put("operator", "uuid"); + map.put("q_characteristics", "TEXT NOT NULL"); + map.put("geo_position", "TEXT NOT NULL"); + map.put("length", "double precision NOT NULL"); + map.put("node_a", "uuid NOT NULL"); + map.put("node_b", "uuid NOT NULL"); + map.put("type", "uuid NOT NULL"); //EVCS + map.put("olm_characteristic", "TEXT NOT NULL"); + map.put("parallel_devices", "int NOT NULL"); + //map.put("parallelDevices", "int NOT NULL"); + map.put("cos_phi_rated", "TEXT NOT NULL"); + map.put("dsm", "bool NOT NULL"); + map.put("e_cons_annual", "double precision NOT NULL"); + map.put("load_profile", "TEXT NOT NULL"); + + map.put("auto_tap", "bool NOT NULL"); + map.put("tap_pos", "int NOT NULL"); + map.put("type", "uuid NOT NULL"); + + map.put("v_ang", "bool NOT NULL"); + map.put("v_mag", "bool NOT NULL"); + map.put("slack", "bool NOT NULL"); + map.put("subnet", "int NOT NULL"); + //map.put("vRated", "double precision NOT NULL"); + map.put("v_rated", "double precision NOT NULL"); + map.put("v_target", "double precision NOT NULL"); + //map.put("vTarget", "double precision NOT NULL"); + map.put("volt_lvl", "TEXT NOT NULL"); + map.put("charging_points", "int NOT NULL"); + map.put("location_type", "TEXT NOT NULL"); + map.put("v_2g_support", "bool NOT NULL"); + //map.put("voltLvl", "TEXT NOT NULL"); + + map.put("albedo", "double precision NOT NULL"); + map.put("azimuth", "double precision NOT NULL"); + map.put("elevation_angle", "double precision NOT NULL"); + map.put("eta_conv", "double precision NOT NULL"); + map.put("k_g", "double precision NOT NULL"); + map.put("k_t", "double precision NOT NULL"); + + map.put("grid_name", "TEXT NOT NULL"); + map.put("grid_uuid", "uuid NOT NULL"); + + + map.put("b_m", "double precision NOT NULL"); + map.put("d_phi", "double precision NOT NULL"); + map.put("d_v", "double precision NOT NULL"); + map.put("g_m", "double precision NOT NULL"); + map.put("r_sc", "double precision NOT NULL"); + map.put("tap_max", "int NOT NULL"); + map.put("tap_min", "int NOT NULL"); + map.put("tap_neutr", "int NOT NULL"); + map.put("tap_side", "bool NOT NULL"); + map.put("v_rated_a", "int NOT NULL"); + map.put("v_rated_b", "int NOT NULL"); + map.put("x_sc", "int NOT NULL"); + map.put("graphic_layer", "TEXT NOT NULL"); + map.put("line", "uuid NOT NULL"); + map.put("path", "TEXT NOT NULL"); + map.put("point", "TEXT NOT NULL"); + map.put("inlet_temp", "double precision NOT NULL"); + map.put("return_temp", "double precision NOT NULL"); + map.put("storage_volume_lvl", "double precision NOT NULL"); + map.put("storage_volume_lvl_min", "double precision NOT NULL"); + map.put("thermal_bus", "uuid NOT NULL"); + map.put("eth_capa", "double precision NOT NULL"); + map.put("eth_losses", "double precision NOT NULL"); + map.put("lower_temperature_limit", "double precision NOT NULL"); + map.put("target_temperature", "double precision NOT NULL"); + map.put("upper_temperature_limit", "double precision NOT NULL"); + map.put("b", "double precision NOT NULL"); + map.put("g", "double precision NOT NULL"); + map.put("i_max", "double precision NOT NULL"); + map.put("r", "double precision NOT NULL"); + map.put("x", "double precision NOT NULL"); + + map.put("connected_assets", "TEXT NOT NULL"); + map.put("capex", "double precision NOT NULL"); + map.put("control_strategy", "TEXT NOT NULL"); + + map.put("input_model", "uuid NOT NULL"); + map.put("soc", "double precision NOT NULL"); + map.put("p_max", "double precision NOT NULL"); + map.put("p_min", "double precision NOT NULL"); + map.put("p_ref", "double precision NOT NULL"); + + map.put("dod", "double precision NOT NULL"); + map.put("e_storage", "double precision NOT NULL"); + map.put("eta", "double precision NOT NULL"); + map.put("life_cycle", "double precision NOT NULL"); + map.put("life_time", "double precision NOT NULL"); + map.put("opex", "double precision NOT NULL"); + map.put("active_power_gradient", "double precision NOT NULL"); + + return map; + } + + public static String quote(String input, String quoteSymbol) { + if (input == "") { + return "NULL"; + } else { + return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index 8b13ec0d0..b3b5d704f 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -64,13 +64,8 @@ public ResultSet executeQuery(Statement stmt, String query) throws SQLException * @param updateQuery the query to execute * @return The number of updates or a negative number if the execution failed */ - public int executeUpdate(String updateQuery) { - try (Statement stmt = getConnection().createStatement()) { - return stmt.executeUpdate(updateQuery); - } catch (SQLException e) { - log.error(String.format("Error at execution of query \"%1.127s\": ", updateQuery), e); - return -1; - } + public int executeUpdate(String updateQuery) throws SQLException { + return getConnection().createStatement().executeUpdate(updateQuery); } /** @@ -160,4 +155,11 @@ public Map extractFieldMap(ResultSet rs) { } return insensitiveFieldsToAttributes; } + + public boolean tableExists(Connection connection, String tableName) throws SQLException { + DatabaseMetaData meta = connection.getMetaData(); + ResultSet resultSet = meta.getTables(null, null, tableName, new String[] {"TABLE"}); + + return resultSet.next(); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java index e59727668..1801ee345 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java @@ -7,12 +7,23 @@ import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.models.UniqueEntity; +import edu.ie3.datamodel.models.timeseries.TimeSeries; +import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; +import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput; +import edu.ie3.datamodel.models.value.Value; + import java.util.Optional; +import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger; + /** A naming strategy for database entities */ public class DatabaseNamingStrategy { private static final String TIME_SERIES_PREFIX = "time_series_"; + + private static final String LOAD_PROFILE_PREFIX = "load_profile_"; + private final EntityPersistenceNamingStrategy entityPersistenceNamingStrategy; public DatabaseNamingStrategy(EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) { @@ -42,7 +53,47 @@ public String getTimeSeriesEntityName(ColumnScheme columnScheme) { return TIME_SERIES_PREFIX + columnScheme.getScheme(); } + /** + * Provides the name of a load profile given by the load profile key + * @param lpKey Load profile key + * @return the table name + */ + private String getLoadProfileEntityName(String lpKey) { + return LOAD_PROFILE_PREFIX + lpKey; + } + + /** + * Provides the name of a unique entity class. + * @param cls Class extends UniqueEntity + * @return the table name + */ public Optional getEntityName(Class cls) { return entityPersistenceNamingStrategy.getEntityName(cls); } + + /** + * Provides the name of a time series. Used to determine the table name in SQL database. + * @param timeSeries + * @return the table name + */ + public , E extends TimeSeriesEntry, V extends Value> + Optional getEntityName(T timeSeries) { + if (timeSeries instanceof IndividualTimeSeries individualTimeSeries) { + Optional maybeFirstElement = individualTimeSeries.getEntries().stream().findFirst(); + if (maybeFirstElement.isPresent()) { + Class valueClass = maybeFirstElement.get().getValue().getClass(); + return Optional.of(getTimeSeriesEntityName(ColumnScheme.parse(valueClass).orElseThrow())); + } else { + logger.error("Unable to determine content of time series {}", timeSeries); + return Optional.empty(); + } + } else if (timeSeries instanceof LoadProfileInput loadProfileInput) { + return Optional.of( + getLoadProfileEntityName(loadProfileInput.getType().getKey()) + ); + } else { + logger.error("There is no naming strategy defined for {}", timeSeries); + return Optional.empty(); + } + } } diff --git a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java index 830fbef02..4d2a48454 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java @@ -24,6 +24,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; import javax.measure.Quantity; @@ -411,7 +412,7 @@ protected String processOperationTime(OperationTime operationTime, String fieldN * @return string representation of the ZonedDateTime */ protected String processZonedDateTime(ZonedDateTime zonedDateTime) { - return zonedDateTime.toString(); + return zonedDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss Z")); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java index 8f3bb42f4..b3257ed98 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java @@ -79,6 +79,18 @@ Try, ProcessorProviderException> handleEntity(T en .transformF(ProcessorProviderException::new)); } + public Set> handleEntities(List entities) throws ProcessorProviderException { + Set setOfEntities = new HashSet<>(entities); + Set> setOfMaps = new HashSet<>(); + for (T entity : setOfEntities) { + LinkedHashMap entryResult = handleEntity(entity).getOrThrow(); + + /* Prepare the actual result and add them to the set of all results */ + setOfMaps.add(new LinkedHashMap<>(entryResult)); + } + return setOfMaps; + } + /** * Get the correct entity processor * diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java new file mode 100644 index 000000000..e1c1c4e0b --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -0,0 +1,507 @@ +package edu.ie3.datamodel.io.sink; + +import edu.ie3.datamodel.exceptions.*; +import edu.ie3.datamodel.io.DatabaseIdentifier; +import edu.ie3.datamodel.io.connectors.SqlConnector; +import edu.ie3.datamodel.io.extractor.Extractor; +import edu.ie3.datamodel.io.extractor.NestedEntity; +import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; +import edu.ie3.datamodel.io.processor.ProcessorProvider; +import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey; +import edu.ie3.datamodel.models.UniqueEntity; +import edu.ie3.datamodel.models.input.*; +import edu.ie3.datamodel.models.input.connector.*; +import edu.ie3.datamodel.models.input.container.GraphicElements; +import edu.ie3.datamodel.models.input.container.JointGridContainer; +import edu.ie3.datamodel.models.input.container.RawGridElements; +import edu.ie3.datamodel.models.input.container.SystemParticipants; +import edu.ie3.datamodel.models.input.graphics.GraphicInput; +import edu.ie3.datamodel.models.input.system.*; +import edu.ie3.datamodel.models.input.thermal.ThermalBusInput; +import edu.ie3.datamodel.models.input.thermal.ThermalUnitInput; +import edu.ie3.datamodel.models.result.ResultEntity; +import edu.ie3.datamodel.models.timeseries.TimeSeries; +import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; +import edu.ie3.datamodel.models.value.Value; +import edu.ie3.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.ie3.datamodel.io.SqlUtils.quote; +import static java.util.UUID.randomUUID; +import static java.util.stream.Collectors.groupingBy; + + +public class SqlSink { + + protected static final Logger log = LoggerFactory.getLogger(SqlSink.class); + + private final SqlConnector connector; + private final DatabaseNamingStrategy databaseNamingStrategy; + private final ProcessorProvider processorProvider; + private final String schemaName; + + private static final String GRID_NAME = "grid_name"; + private static final String GRID_UUID = "grid_uuid"; + private static final String TIME_SERIES = "time_series"; + + public SqlSink( + String schemaName, + DatabaseNamingStrategy databaseNamingStrategy, + SqlConnector connector + ) throws EntityProcessorException { + this(schemaName, new ProcessorProvider(), databaseNamingStrategy, connector); + } + + public SqlSink( + String schemaName, + ProcessorProvider processorProvider, + DatabaseNamingStrategy databaseNamingStrategy, + SqlConnector connector + ) { + this.connector = connector; + this.databaseNamingStrategy = databaseNamingStrategy; + this.processorProvider = processorProvider; + this.schemaName = schemaName; + } + + public void shutdown() { + connector.shutdown(); + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + public void persistAll(Collection entities, DatabaseIdentifier identifier) throws SQLException { + // Extract nested entities and add them to the set of entities + Set entitiesToAdd = new HashSet<>(entities); // entities to persist + entities.stream().filter( + entity -> entity instanceof NestedEntity + ).forEach( + entity -> { + try { + entitiesToAdd.addAll((List) Extractor.extractElements((NestedEntity) entity).stream().toList()); + } catch (ExtractorException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + entity.getClass()), + e); + } + }); + + // Persist the entities in hierarchic order to avoid failure because of foreign keys + for (Class cls : hierarchicInsert()) { + persistMixedList( + entitiesToAdd.stream().filter( + ent -> cls.isAssignableFrom(ent.getClass()) + ).collect(Collectors.toList()), identifier); + entitiesToAdd.removeIf(ent -> cls.isAssignableFrom(ent.getClass())); // maybe it's not necessary but I'm not sure if there are entities who aren't in the hierarchic structure + } + persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities + } + + public void persistAllIgnoreNested(Collection entities, DatabaseIdentifier identifier) { + persistMixedList(new ArrayList<>(entities), identifier); + } + + public void persist(C entity, DatabaseIdentifier identifier) throws SQLException { + if (entity instanceof InputEntity inputEntity) { + persistIncludeNested(inputEntity, identifier); + } else if (entity instanceof ResultEntity resultEntity) { + insert(resultEntity, identifier); + } else if (entity instanceof TimeSeries timeSeries) { + persistTimeSeries(timeSeries, identifier); + } else { + log.error("I don't know how to handle an entity of class {}", entity.getClass().getSimpleName()); + } + } + + public void persistIgnoreNested(C entity, DatabaseIdentifier identifier) throws SQLException { + insert(entity, identifier); + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + protected void persistListIncludeNested(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + if (NestedEntity.class.isAssignableFrom(cls)) { + entities.forEach( + entity -> { + try { + List arr = new ArrayList<>(Extractor.extractElements((NestedEntity) entity)); + persistAll(arr, identifier); + } catch (ExtractorException | SQLException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + cls.getSimpleName()), + e); + } + } + ); + insertListIgnoreNested(entities, cls, identifier); + } else { + insertListIgnoreNested(entities, cls, identifier); + } + } + + public void persistIncludeNested(C entity, DatabaseIdentifier identifier) throws SQLException { + Set entitiesToAdd = new HashSet<>(); + entitiesToAdd.add(entity); + persistAll(entitiesToAdd, identifier); + } + + private void persistMixedList(List entities, DatabaseIdentifier identifier) { + Map, List> entitiesPerClass = entities.stream() + .collect(groupingBy(entity -> (Class) entity.getClass())); + entitiesPerClass.forEach( + (cls, ent) -> { + try { + persistList(ent, cls, identifier); + } catch (SQLException e) { + log.error(String.format( + "An error occurred during extraction of entity'%s': ", + cls.getSimpleName()), + e); + throw new RuntimeException(String.format( + "An error occurred during extraction of entity '%s', SQLReason: '%s'", + cls.getSimpleName(), e.getMessage()), e); + } + } + ); + } + + private , V extends Value> void persistList(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + // Check if there are only elements of the same class + Class firstClass = entities.get(0).getClass(); + boolean allSameClass = true; + for (Object obj : entities) { + if (obj.getClass() != firstClass) { + allSameClass = false; + break; + } + } + + if (allSameClass) { + if (InputEntity.class.isAssignableFrom(cls)) { + insertListIgnoreNested(entities, cls, identifier); + } else if (ResultEntity.class.isAssignableFrom(cls)) { + insertListIgnoreNested(entities, cls, identifier); + } else if (TimeSeries.class.isAssignableFrom(cls)) { + persistListOfTimeSeries((List>) entities, identifier); + } else { + log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); + } + } else { + log.error("The list isn't homogenous regarding the classes of the elements."); + } + } + + /** + * Writes a list of entities into a sql table. It's necessary that all entities have the same class. + */ + private void insertListIgnoreNested(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + try { + String[] headerElements = StringUtils.camelCaseToSnakeCase(processorProvider.getHeaderElements(cls)); + String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); + query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); + connector.executeUpdate(query); + //int expectedElements = entities.size(); + //int addedElements = connector.executeUpdate(query); + //System.out.println("Insert " + cls.getSimpleName() + ": " + " expected " + expectedElements + ", added " + addedElements); + } catch (ProcessorProviderException e) { + log.error("Exception occurred during processor request: ", e); + } + } + + private , V extends Value> void persistListOfTimeSeries( + List> list, DatabaseIdentifier identifier + ) throws SQLException { + for (TimeSeries ts : list) { + persistTimeSeries(ts, identifier); + } + } + + + protected , V extends Value> void persistTimeSeries(TimeSeries timeSeries, DatabaseIdentifier identifier) throws SQLException { + try { + TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); + String[] headerElements = processorProvider.getHeaderElements(key); + persistTimeSeries(timeSeries, headerElements, identifier); + } catch (ProcessorProviderException e) { + log.error( + "Exception occurred during receiving of header elements. Cannot write this element.", e); + } catch (IOException e) { + log.error("Exception occurred during closing of writer.", e); + } + } + + private , V extends Value> void persistTimeSeries( + TimeSeries timeSeries, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException, IOException, SQLException { + try { + String query = basicInsertQueryValuesITS(schemaName, databaseNamingStrategy.getEntityName(timeSeries).orElseThrow(), headerElements); + Set> entityFieldData = + processorProvider.handleTimeSeries(timeSeries); + query = query + entityFieldData.stream().map( + //data -> createTimeSeriesQueryLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", "ON CONFLICT (uuid) DO NOTHING;")); + data -> queryTimeSeriesValueLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", ";")); + try { + connector.executeUpdate(query); + //int expectedElements = timeSeries.getEntries().size(); + //int addedElements = connector.executeUpdate(query); + //System.out.println("Insert " + "TimeSeries" + ": " + " expected " + expectedElements + ", added " + addedElements); + } catch (SQLException e) { + throw new SQLException(String.format("Error at execution of query \"%1.127s\": ", query), e); + } + } catch (ProcessorProviderException e) { + throw new ProcessorProviderException("Exception occurred during processor request: ", e); + } + } + + public void persistJointGrid(JointGridContainer jointGridContainer) throws SQLException { + DatabaseIdentifier identifier = new DatabaseIdentifier( + jointGridContainer.getGridName(), randomUUID() + ); + + // get raw grid entities with types or operators + RawGridElements rawGridElements = jointGridContainer.getRawGrid(); + Set nodes = rawGridElements.getNodes(); + Set lines = rawGridElements.getLines(); + Set transformer2Ws = rawGridElements.getTransformer2Ws(); + Set transformer3Ws = rawGridElements.getTransformer3Ws(); + Set switches = rawGridElements.getSwitches(); + Set measurementUnits = rawGridElements.getMeasurementUnits(); + + // get system participants with types or operators + SystemParticipants systemParticipants = jointGridContainer.getSystemParticipants(); + Set bmPlants = systemParticipants.getBmPlants(); + Set chpPlants = systemParticipants.getChpPlants(); + Set evCS = systemParticipants.getEvCS(); + Set evs = systemParticipants.getEvs(); + Set fixedFeedIns = systemParticipants.getFixedFeedIns(); + Set heatPumps = systemParticipants.getHeatPumps(); + Set loads = systemParticipants.getLoads(); + Set pvPlants = systemParticipants.getPvPlants(); + Set storages = systemParticipants.getStorages(); + Set wecPlants = systemParticipants.getWecPlants(); + Set emSystems = systemParticipants.getEmSystems(); + + // get graphic elements (just for better readability, we could also just get them directly + // below) + GraphicElements graphicElements = jointGridContainer.getGraphics(); + + // extract types + Set types = + Stream.of( + lines, + transformer2Ws, + transformer3Ws, + bmPlants, + chpPlants, + evs, + heatPumps, + storages, + wecPlants) + .flatMap(Collection::stream) + .map(Extractor::extractType) + .collect(Collectors.toSet()); + + // extract operators + Set operators = + Stream.of( + nodes, + lines, + transformer2Ws, + transformer3Ws, + switches, + measurementUnits, + bmPlants, + chpPlants, + evCS, + evs, + fixedFeedIns, + heatPumps, + loads, + pvPlants, + storages, + wecPlants, + emSystems) + .flatMap(Collection::stream) + .map(Extractor::extractOperator) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + + List toAdd = new LinkedList<>(); + toAdd.addAll(rawGridElements.allEntitiesAsList()); + toAdd.addAll(systemParticipants.allEntitiesAsList()); + toAdd.addAll(graphicElements.allEntitiesAsList()); + toAdd.addAll(types); + toAdd.addAll(operators); + + // persist all entities + persistAll(toAdd, identifier); + } + private void insert(C entity, DatabaseIdentifier identifier) throws SQLException { + try { + String[] headerElements = StringUtils.camelCaseToSnakeCase(processorProvider.getHeaderElements(entity.getClass())); + String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(entity.getClass()).orElseThrow(), headerElements) + queryValueLine(entity, headerElements, identifier) + ";"; + connector.executeUpdate(query); + } catch (ProcessorProviderException e) { + log.error("Exception occurred during receiving of header elements. Cannot write this element.", e); + } + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + /** + * Provides the value lists for an insertion query. Conflicts because of the primary key uuid will be ignored. + * + * WARNING: It's assumed that all entities are from the same class C. + */ + private String createInsertQueryBodyIgnoreConflict(List entities, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { + Set> entityFieldData = + processorProvider + .handleEntities(entities); + String queryBody = ""; + queryBody = queryBody + entityFieldData.stream().map( + data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)).collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + return queryBody; + } + + /** + * Provides the value lists for an insertion query. + * + * WARNING: It's assumed that all entities are from the same class C. + */ + private String createInsertQueryBody(List entities, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { + Set> entityFieldData = + processorProvider + .handleEntities(entities); + String queryBody = ""; + queryBody = queryBody + entityFieldData.stream().map( + data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)).collect(Collectors.joining(",\n", "", ";")); + return queryBody; + } + + /** + * Creates a line with the values of one entity for an insertion query using the entityFieldData. + */ + private String queryValueLine(LinkedHashMap entityFieldData, String[] headerElements, DatabaseIdentifier identifier) { + return writeOneLine(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery())); + } + + /** + * Creates a line with the values of one entity for an insertion query. + */ + private String queryValueLine(C entity, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { + LinkedHashMap entityFieldData = + processorProvider + .handleEntity(entity) + .map(this::sqlEntityFieldData) + .getOrThrow(); + return writeOneLine(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery())); + } + + private String queryTimeSeriesValueLine(Map entityFieldData, String[] headerElements, DatabaseIdentifier identifier, String TSuuid) { + return writeOneLine(Stream.concat(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery()), Stream.of(quote(TSuuid,"'")))); + } + + /** + * Quote a given entity field + */ + private LinkedHashMap sqlEntityFieldData( + LinkedHashMap entityFieldData) { + + return entityFieldData.entrySet().stream() + .map( + mapEntry -> + new AbstractMap.SimpleEntry<>( + mapEntry.getKey(), + quote(mapEntry.getValue(), "'"))) + .collect( + Collectors.toMap( + AbstractMap.SimpleEntry::getKey, + AbstractMap.SimpleEntry::getValue, + (v1, v2) -> { + throw new IllegalStateException( + "Converting entity data to RFC 4180 compliant strings has lead to duplicate keys. Initial input:\n\t" + + entityFieldData.entrySet().stream() + .map(entry -> entry.getKey() + " = " + entry.getValue()) + .collect(Collectors.joining(",\n\t"))); + }, + LinkedHashMap::new)); + } + + + /** + * "INSERT INTO" line with schemaName.tableName + */ + private static String basicInsertQuery(String schemaName, String tableName) { + return "INSERT INTO\n\t" + schemaName + "." + tableName; + } + + /** + * Provides the insert, column names, grid identifier and the VALUES statement for a query. + */ + private String basicInsertQueryValuesGrid(String schemaName, String tableName, String[] headerElements) { + String[] addParams = {GRID_NAME, GRID_UUID}; + return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(headerElements, addParams) + "\nVALUES\n"; + } + + /** + * Provides the insert, column names, grid identifier, time_series uuid and the VALUES statement for a query. + */ + private String basicInsertQueryValuesITS(String schemaName, String tableName, String[] headerElements) { + String[] addParams = {GRID_NAME, GRID_UUID, TIME_SERIES}; + return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(headerElements, addParams) + "\nVALUES\n"; + } + + /** + * Converts a stream of strings into an one line string with brackets. + */ + private String writeOneLine(Stream entries) { + return "(" + entries.collect(Collectors.joining(",")) + ")"; + } + + /** + * Converts an array of strings and an array of strings (for additional parameters) into an one line string with brackets. + */ + private String writeOneLine(String[] entries, String[] addParams) { + return writeOneLine(Stream.concat(Arrays.stream(entries), Arrays.stream(addParams))); + } + + + /** + * @return insertion order for unique entities + */ + private static List> hierarchicInsert() { + List> sortedInsert = new ArrayList<>(); + sortedInsert.add(AssetTypeInput.class); //1. Types + sortedInsert.add(OperatorInput.class); //2. Operators + sortedInsert.add(NodeInput.class); //3. Nodes + sortedInsert.add(ThermalBusInput.class); //4. ThermalBus + sortedInsert.add(ThermalUnitInput.class); //5. ThermalUnit + sortedInsert.add(ConnectorInput.class); //6a. ConnectorInput + sortedInsert.add(SystemParticipantInput.class); //6b. SystemParticipantInput + sortedInsert.add(GraphicInput.class); //7. GraphicInput + //8. Rest + return sortedInsert; + } + + private String entitiesToString(Collection entities) { + String res = "[\n"; + int no = 0; + Iterator iterator = entities.iterator(); + while (iterator.hasNext()) { + res = res + "\t (" + no + ") " + iterator.next().toString() + "\n"; + no++; + } + res = res + "]"; + return res; + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy index 62491ebc6..84f0e6ca5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy @@ -312,7 +312,7 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { testBaseFolderPath.resolve("line_input.csv").toFile().exists() testBaseFolderPath.resolve("line_type_input.csv").toFile().exists() testBaseFolderPath.resolve("load_input.csv").toFile().exists() - testBaseFolderPath.resolve( "node_input.csv").toFile().exists() + testBaseFolderPath.resolve("node_input.csv").toFile().exists() testBaseFolderPath.resolve("operator_input.csv").toFile().exists() testBaseFolderPath.resolve("pv_input.csv").toFile().exists() testBaseFolderPath.resolve("storage_input.csv").toFile().exists() diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy new file mode 100644 index 000000000..b8759a546 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -0,0 +1,402 @@ +package edu.ie3.datamodel.io.sink + +import edu.ie3.datamodel.io.DatabaseIdentifier +import edu.ie3.datamodel.io.SqlUtils +import edu.ie3.datamodel.io.connectors.SqlConnector +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory +import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation +import edu.ie3.datamodel.io.processor.ProcessorProvider +import edu.ie3.datamodel.io.processor.input.InputEntityProcessor +import edu.ie3.datamodel.io.processor.result.ResultEntityProcessor +import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessor +import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey +import edu.ie3.datamodel.io.source.csv.CsvTimeSeriesSource +import edu.ie3.datamodel.io.source.sql.SqlDataSource +import edu.ie3.datamodel.io.source.sql.SqlTimeSeriesSource +import edu.ie3.datamodel.models.OperationTime +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.input.NodeInput +import edu.ie3.datamodel.models.input.OperatorInput +import edu.ie3.datamodel.models.input.connector.LineInput +import edu.ie3.datamodel.models.input.connector.Transformer2WInput +import edu.ie3.datamodel.models.input.connector.type.LineTypeInput +import edu.ie3.datamodel.models.input.connector.type.Transformer2WTypeInput +import edu.ie3.datamodel.models.input.graphics.LineGraphicInput +import edu.ie3.datamodel.models.input.graphics.NodeGraphicInput +import edu.ie3.datamodel.models.input.system.EmInput +import edu.ie3.datamodel.models.input.system.EvcsInput +import edu.ie3.datamodel.models.input.system.LoadInput +import edu.ie3.datamodel.models.input.system.PvInput +import edu.ie3.datamodel.models.input.system.StorageInput +import edu.ie3.datamodel.models.input.system.WecInput +import edu.ie3.datamodel.models.input.system.characteristic.CosPhiFixed +import edu.ie3.datamodel.models.input.system.type.StorageTypeInput +import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput +import edu.ie3.datamodel.models.input.thermal.ThermalBusInput +import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput +import edu.ie3.datamodel.models.result.system.EmResult +import edu.ie3.datamodel.models.result.system.EvResult +import edu.ie3.datamodel.models.result.system.EvcsResult +import edu.ie3.datamodel.models.result.system.FlexOptionsResult +import edu.ie3.datamodel.models.result.system.PvResult +import edu.ie3.datamodel.models.result.system.WecResult +import edu.ie3.datamodel.models.timeseries.TimeSeries +import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput +import edu.ie3.datamodel.models.value.EnergyPriceValue +import edu.ie3.datamodel.models.value.Value +import edu.ie3.test.common.GridTestData +import edu.ie3.test.common.SampleJointGrid +import edu.ie3.test.common.ThermalUnitInputTestData +import edu.ie3.test.common.TimeSeriesTestData +import edu.ie3.test.helper.TestContainerHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.io.FileIOUtils +import org.jetbrains.annotations.NotNull +import org.testcontainers.containers.Container +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification +import org.testcontainers.spock.Testcontainers +import tech.units.indriya.quantity.Quantities + +import javax.measure.Quantity +import javax.measure.quantity.Power +import java.nio.file.Path + +import edu.ie3.test.common.SystemParticipantTestData + +import java.sql.SQLException +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + +import static edu.ie3.datamodel.models.StandardUnits.ENERGY_PRICE +import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM +import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM +import static edu.ie3.util.quantities.PowerSystemUnits.KILOVOLTAMPERE +import static tech.units.indriya.unit.Units.PERCENT + +@Testcontainers +class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeriesTestData { + + @Shared + PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:14.2") + + @Shared + SqlConnector connector + + @Shared + SqlDataSource sqlSource + + @Shared + DatabaseNamingStrategy namingStrategy + + @Shared + DatabaseIdentifier identifier + + static String schemaName = "public" + + def setupSpec() { + // Copy sql import scripts into docker + MountableFile sqlImportFile = getMountableFile("_sql/") + postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/") + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "setup.sql") + + connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) + + namingStrategy = new DatabaseNamingStrategy() + + identifier = new DatabaseIdentifier("vn_simona", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120002")) + + sqlSource = new SqlDataSource(connector, schemaName, namingStrategy) + } + + def setup() { + // Execute import script + Iterable importFiles = Arrays.asList( + "types.sql", + "result_entities.sql", + "input_entities.sql", + "time_series.sql", + "load_profile.sql" + ) + for (String file: importFiles) { + Container.ExecResult res = postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + file) + assert res.stderr.empty + } + } + + def cleanup() { + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "cleanup.sql") + } + + def "SQL sink can persist provided elements correctly"() { + given: + TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + HashMap timeSeriesProcessorMap = new HashMap<>() + timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) + IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries + + SqlSink sink = new SqlSink(schemaName, + new ProcessorProvider([ + new ResultEntityProcessor(PvResult), + new ResultEntityProcessor(WecResult), + new ResultEntityProcessor(EvResult), + new ResultEntityProcessor(EvcsResult), + new ResultEntityProcessor(EmResult), + new ResultEntityProcessor(FlexOptionsResult), + new InputEntityProcessor(Transformer2WInput), + new InputEntityProcessor(NodeInput), + new InputEntityProcessor(EvcsInput), + new InputEntityProcessor(Transformer2WTypeInput), + new InputEntityProcessor(LineGraphicInput), + new InputEntityProcessor(NodeGraphicInput), + new InputEntityProcessor(CylindricalStorageInput), + new InputEntityProcessor(ThermalHouseInput), + new InputEntityProcessor(OperatorInput), + new InputEntityProcessor(LineInput), + new InputEntityProcessor(ThermalBusInput), + new InputEntityProcessor(LineTypeInput), + new InputEntityProcessor(LoadInput), + new InputEntityProcessor(EmInput) + ], timeSeriesProcessorMap), + namingStrategy, + connector) + UUID uuid = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") + UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") + Quantity p = Quantities.getQuantity(10, StandardUnits.ACTIVE_POWER_IN) + Quantity q = Quantities.getQuantity(10, StandardUnits.REACTIVE_POWER_IN) + PvResult pvResult = new PvResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + WecResult wecResult = new WecResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + EvcsResult evcsResult = new EvcsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + EmResult emResult = new EmResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + + Quantity pRef = Quantities.getQuantity(5.1, StandardUnits.ACTIVE_POWER_RESULT) + Quantity pMin = Quantities.getQuantity(-6, StandardUnits.ACTIVE_POWER_RESULT) + Quantity pMax = Quantities.getQuantity(6, StandardUnits.ACTIVE_POWER_RESULT) + FlexOptionsResult flexOptionsResult = new FlexOptionsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, pRef, pMin, pMax) + + when: + sink.persistAll([ + pvResult, + wecResult, + evcsResult, + emResult, + flexOptionsResult, + GridTestData.transformerCtoG, + GridTestData.lineGraphicCtoD, + GridTestData.nodeGraphicC, + ThermalUnitInputTestData.cylindricStorageInput, + ThermalUnitInputTestData.thermalHouseInput, + SystemParticipantTestData.evcsInput, + SystemParticipantTestData.loadInput, + SystemParticipantTestData.emInput, + individualTimeSeries + ], identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "evcs_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "flex_options_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 4 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "cylindrical_storage_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_graphic_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_graphic_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_bus_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_house_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "ev_res", ps -> {}).count() == 0 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + + def "A SqlSink can persist a time series."() { + given: + TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + HashMap timeSeriesProcessorMap = new HashMap<>() + timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) + IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries + SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persist(individualTimeSeries, identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink persists a bunch of time series correctly"() { + given: + SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) + SqlDataSource source = new SqlDataSource(connector, schemaName, namingStrategy) + + when: + sink.persistAll(allTimeSeries, identifier) + + then: + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_p", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pq", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_ph", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pqh", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_weather", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink throws an exception if an entity has null for a not null attribute."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + def nestedInput = new PvInput( + UUID.fromString("d56f15b7-8293-4b98-b5bd-58f6273ce229"), + "test_pvInput", + OperatorInput.NO_OPERATOR_ASSIGNED, + OperationTime.notLimited(), + Mock(NodeInput), + new CosPhiFixed("cosPhiFixed:{(0.0,0.95)}"), + 0.2, + Quantities.getQuantity(-8.926613807678223, DEGREE_GEOM), + Quantities.getQuantity(95d, PERCENT), + Quantities.getQuantity(41.01871871948242, DEGREE_GEOM), + 0.8999999761581421, + 1, + false, + Quantities.getQuantity(25d, KILOVOLTAMPERE), + 0.95 + ) + + when: + sink.persistIgnoreNested(nestedInput, identifier) + + then: + def exception = thrown(SQLException) + exception.message == "ERROR: invalid input syntax for type uuid: \"null\"\n Position: 365" + + cleanup: + sink.shutdown() + } + + + def "A valid SqlSink refuses to persist an entity, if no processor can be found for a specific input"() { + given: + def sink = new SqlSink( + schemaName, + new ProcessorProvider( + ProcessorProvider.allEntityProcessors(), + new HashMap, Value>, TimeSeriesEntry, Value>>()), + namingStrategy, + connector) + + when: + sink.persist(individualEnergyPriceTimeSeries, identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 0 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink throws an exception if a nested entity hasn't all of its nested entity."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persistIgnoreNested(SystemParticipantTestData.loadInput, identifier) + + then: + def exception = thrown(SQLException) + exception.message == "ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + + " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." + + cleanup: + sink.shutdown() + } + + + def "A valid SqlSink should persist a valid joint grid container correctly"() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persistJointGrid(SampleJointGrid.grid()) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 6 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_type_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 7 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_type_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 2 + + cleanup: + sink.shutdown() + } + + /* + def "Ausgabe"() { + given: + System.out.println(SqlUtils.getDataTypes(StorageTypeInput.class)) + //System.out.println(SqlUtils.getDataTypes(StorageInput.class)) + //System.out.println(SqlUtils.getDataTypes(NodeInput.class)) + + System.out.println(SqlUtils.getDataTypes(PvResult.class)) + System.out.println(SqlUtils.getDataTypes(WecResult.class)) + System.out.println(SqlUtils.getDataTypes(EvResult.class)) + System.out.println(SqlUtils.getDataTypes(EvcsResult.class)) + System.out.println(SqlUtils.getDataTypes(EmResult.class)) + System.out.println(SqlUtils.getDataTypes(FlexOptionsResult.class)) + + + System.out.println(SqlUtils.getDataTypes(Transformer2WInput.class)) + System.out.println(SqlUtils.getDataTypes(NodeInput.class)) + System.out.println(SqlUtils.getDataTypes(EvcsInput.class)) + System.out.println(SqlUtils.getDataTypes(LineGraphicInput.class)) + System.out.println(SqlUtils.getDataTypes(NodeGraphicInput.class)) + System.out.println(SqlUtils.getDataTypes(CylindricalStorageInput.class)) + System.out.println(SqlUtils.getDataTypes(ThermalHouseInput.class)) + System.out.println(SqlUtils.getDataTypes(OperatorInput.class)) + System.out.println(SqlUtils.getDataTypes(LineInput.class)) + System.out.println(SqlUtils.getDataTypes(ThermalBusInput.class)) + System.out.println(SqlUtils.getDataTypes(LineTypeInput.class)) + System.out.println(SqlUtils.getDataTypes(LoadInput.class)) + System.out.println(SqlUtils.getDataTypes(EmInput.class)) + + System.out.println(SqlUtils.getDataTypes(Transformer2WTypeInput.class)) + + + when: + def nummer = 1 + + then: + nummer == 1 + } + */ +} diff --git a/src/test/groovy/edu/ie3/test/common/SampleJointGrid.groovy b/src/test/groovy/edu/ie3/test/common/SampleJointGrid.groovy index a9f1aeb63..57a19b06f 100644 --- a/src/test/groovy/edu/ie3/test/common/SampleJointGrid.groovy +++ b/src/test/groovy/edu/ie3/test/common/SampleJointGrid.groovy @@ -381,8 +381,8 @@ class SampleJointGrid extends SystemParticipantTestData { public static final Transformer2WTypeInput transformerType_MV_HV_110KV = new Transformer2WTypeInput( - UUID.fromString("08559390-d7c0-4427-a2dc-97ba312ae0ac"), - "MS-NS_1", + UUID.fromString("deecabd2-5ddb-11ee-8c99-0242ac120002"), + "HS-MS_1", Quantities.getQuantity(10.078, OHM), Quantities.getQuantity(23.312, OHM), Quantities.getQuantity(800d, KILOVOLTAMPERE), diff --git a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy index 61986f7ad..0da55b6cd 100644 --- a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy @@ -5,6 +5,8 @@ */ package edu.ie3.test.common +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.IntValue @@ -52,6 +54,11 @@ trait TimeSeriesTestData { ] as Set ) + IndividualTimeSeriesMetaInformation individualEnergyPriceTimeSeriesMeta = new IndividualTimeSeriesMetaInformation( + UUID.fromString("a4bbcb77-b9d0-4b88-92be-b9a14a3e332b"), + ColumnScheme.ENERGY_PRICE + ) + Set> individualEnergyPriceTimeSeriesProcessed = [ [ "uuid" : "9e4dba1b-f3bb-4e40-bd7e-2de7e81b7704", diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/cleanup.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/cleanup.sql new file mode 100644 index 000000000..3dd1acf4a --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/cleanup.sql @@ -0,0 +1,7 @@ +DO $$ DECLARE +r RECORD; +BEGIN +FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP + EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; +END LOOP; +END $$; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql new file mode 100644 index 000000000..fa6d569aa --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql @@ -0,0 +1,240 @@ +CREATE TABLE public.node_input +( + uuid uuid PRIMARY KEY, + geo_position TEXT NOT NULL, + id TEXT NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + slack bool NOT NULL, + subnet int NOT NULL, + v_rated double precision NOT NULL, + v_target double precision NOT NULL, + volt_lvl TEXT NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.transformer_2_w_input +( + uuid uuid PRIMARY KEY, + auto_tap bool NOT NULL, + id TEXT NOT NULL, + node_a uuid NOT NULL REFERENCES node_input(uuid), + node_b uuid NOT NULL REFERENCES node_input(uuid), + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + parallel_devices int NOT NULL, + tap_pos int NOT NULL, + type uuid NOT NULL REFERENCES transformer_2_w_type_input(uuid), + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.evcs_input +( + uuid uuid PRIMARY KEY, + charging_points int NOT NULL, + cos_phi_rated TEXT NOT NULL, + id TEXT NOT NULL, + location_type TEXT NOT NULL, + node uuid NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + q_characteristics TEXT NOT NULL, + type TEXT NOT NULL, + v_2g_support bool NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.line_graphic_input +( + uuid uuid PRIMARY KEY, + graphic_layer TEXT NOT NULL, + line uuid NOT NULL, + path TEXT, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.node_graphic_input +( + uuid uuid PRIMARY KEY, + graphic_layer TEXT NOT NULL, + node uuid NOT NULL, + path TEXT, + point TEXT NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.cylindrical_storage_input +( + uuid uuid PRIMARY KEY, + c double precision NOT NULL, + id TEXT NOT NULL, + inlet_temp double precision NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + return_temp double precision NOT NULL, + storage_volume_lvl double precision NOT NULL, + storage_volume_lvl_min double precision NOT NULL, + thermal_bus uuid NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.thermal_house_input +( + uuid uuid PRIMARY KEY, + eth_capa double precision NOT NULL, + eth_losses double precision NOT NULL, + id TEXT NOT NULL, + lower_temperature_limit double precision NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + target_temperature double precision NOT NULL, + thermal_bus uuid NOT NULL, + upper_temperature_limit double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.operator_input +( + uuid uuid PRIMARY KEY, + id TEXT NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.line_input +( + uuid uuid PRIMARY KEY, + geo_position TEXT NOT NULL, + id TEXT NOT NULL, + length double precision NOT NULL, + node_a uuid NOT NULL, + node_b uuid NOT NULL, + olm_characteristic TEXT NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + parallel_devices int NOT NULL, + type uuid NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.thermal_bus_input +( + uuid uuid PRIMARY KEY, + id TEXT NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.load_input +( + uuid uuid PRIMARY KEY, + cos_phi_rated TEXT NOT NULL, + dsm bool NOT NULL, + e_cons_annual double precision NOT NULL, + id TEXT NOT NULL, + load_profile TEXT NOT NULL, + node uuid NOT NULL REFERENCES node_input(uuid), + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + q_characteristics TEXT NOT NULL, + s_rated double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.em_input +( + uuid uuid PRIMARY KEY, + connected_assets TEXT NOT NULL, + control_strategy TEXT NOT NULL, + id TEXT NOT NULL, + node uuid NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + q_characteristics TEXT NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.pv_input +( + uuid uuid PRIMARY KEY, + albedo double precision NOT NULL, + azimuth double precision NOT NULL, + cos_phi_rated TEXT NOT NULL, + elevation_angle double precision NOT NULL, + eta_conv double precision NOT NULL, + id TEXT NOT NULL, + k_g double precision NOT NULL, + k_t double precision NOT NULL, + market_reaction bool NOT NULL, + node uuid NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + q_characteristics TEXT NOT NULL, + s_rated double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.storage_input +( + uuid uuid PRIMARY KEY, + id TEXT NOT NULL, + node uuid NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + q_characteristics TEXT NOT NULL, + type uuid NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql new file mode 100644 index 000000000..45cf68173 --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql @@ -0,0 +1,13 @@ +CREATE TABLE public.load_profile_g2 +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + dayOfWeek TEXT NOT NULL, + quarterHourOfDay TEXT NOT NULL, + p double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql new file mode 100644 index 000000000..f6ad68d49 --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql @@ -0,0 +1,79 @@ +CREATE TABLE public.pv_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p double precision NOT NULL, + q double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.wec_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p double precision NOT NULL, + q double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.ev_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p double precision NOT NULL, + q double precision NOT NULL, + soc double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.evcs_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p double precision NOT NULL, + q double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.em_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p double precision NOT NULL, + q double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.flex_options_res +( + uuid uuid PRIMARY KEY, + input_model uuid NOT NULL, + p_max double precision NOT NULL, + p_min double precision NOT NULL, + p_ref double precision NOT NULL, + time timestamp with time zone NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/setup.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/setup.sql new file mode 100644 index 000000000..aacd7775c --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/setup.sql @@ -0,0 +1 @@ +CREATE SCHEMA public; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql new file mode 100644 index 000000000..a5cb071e5 --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql @@ -0,0 +1,16 @@ +CREATE TABLE public.test_data_sql +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + p double precision NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX test_data_sql_series_id ON test_data_sql USING hash (time_series); + +-- Order of columns is important when using btree: https://www.postgresql.org/docs/14/indexes-multicolumn.html +-- Column time_series needs to placed as the first argument since we at most use an equality constraint on +-- time_series and a range query on time. +CREATE UNIQUE INDEX test_data_sql_series_time ON test_data_sql USING btree (time_series, time); \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql new file mode 100644 index 000000000..23184d899 --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql @@ -0,0 +1,123 @@ +CREATE TABLE public.time_series_c +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + price double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_c_series_id ON time_series_c USING hash (time_series); + +-- Order of columns is important when using btree: https://www.postgresql.org/docs/14/indexes-multicolumn.html +-- Column time_series needs to placed as the first argument since we at most use an equality constraint on +-- time_series and a range query on time. +CREATE UNIQUE INDEX time_series_c_series_time ON time_series_c USING btree (time_series, time); + +CREATE TABLE public.time_series_p +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + p double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_p_series_id ON time_series_p USING hash (time_series); + +CREATE UNIQUE INDEX time_series_p_series_time ON time_series_p USING btree (time_series, time); + +CREATE TABLE public.time_series_pq +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + p double precision, + q double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_pq_series_id ON time_series_pq USING hash (time_series); + +CREATE UNIQUE INDEX time_series_pq_series_time ON time_series_pq USING btree (time_series, time); + +CREATE TABLE public.time_series_h +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + heatDemand double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_h_series_id ON time_series_h USING hash (time_series); + +CREATE UNIQUE INDEX time_series_h_series_time ON time_series_h USING btree (time_series, time); + +CREATE TABLE public.time_series_ph +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + p double precision, + heatDemand double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_ph_series_id ON time_series_ph USING hash (time_series); + +CREATE UNIQUE INDEX time_series_ph_series_time ON time_series_ph USING btree (time_series, time); + +CREATE TABLE public.time_series_pqh +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + time timestamp with time zone NOT NULL, + p double precision, + q double precision, + heatDemand double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_pqh_series_id ON time_series_pqh USING hash (time_series); + +CREATE UNIQUE INDEX time_series_pqh_series_time ON time_series_pqh USING btree (time_series, time); + +CREATE TABLE public.time_series_weather +( + uuid uuid PRIMARY KEY, + time_series uuid NOT NULL, + coordinate TEXT NOT NULL, + time timestamp with time zone NOT NULL, + diffuseIrradiance double precision, + directIrradiance double precision, + direction double precision, + temperature double precision, + velocity double precision, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE INDEX time_series_weather_series_id ON time_series_weather USING hash (time_series); + +CREATE UNIQUE INDEX time_series_weather_series_time ON time_series_weather USING btree (time_series, time); \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql new file mode 100644 index 000000000..3d8bd328d --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql @@ -0,0 +1,59 @@ +CREATE TABLE public.transformer_2_w_type_input +( + uuid uuid PRIMARY KEY, + b_m double precision NOT NULL, + d_phi double precision NOT NULL, + d_v double precision NOT NULL, + g_m double precision NOT NULL, + id TEXT NOT NULL, + r_sc double precision NOT NULL, + s_rated double precision NOT NULL, + tap_max int NOT NULL, + tap_min int NOT NULL, + tap_neutr int NOT NULL, + tap_side bool NOT NULL, + v_rated_a double precision NOT NULL, + v_rated_b double precision NOT NULL, + x_sc double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.line_type_input +( + uuid uuid PRIMARY KEY, + b double precision NOT NULL, + g double precision NOT NULL, + i_max double precision NOT NULL, + id TEXT NOT NULL, + r double precision NOT NULL, + v_rated double precision NOT NULL, + x double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +CREATE TABLE public.storage_type_input +( + uuid uuid PRIMARY KEY, + active_power_gradient double precision NOT NULL, + capex double precision NOT NULL, + cos_phi_rated TEXT NOT NULL, + dod double precision NOT NULL, + e_storage double precision NOT NULL, + eta double precision NOT NULL, + id TEXT NOT NULL, + life_cycle double precision NOT NULL, + life_time double precision NOT NULL, + opex double precision NOT NULL, + p_max double precision NOT NULL, + s_rated double precision NOT NULL, + grid_name TEXT NOT NULL, + grid_uuid uuid NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql b/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql index 13bb195bf..56251bb93 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql +++ b/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql @@ -1,6 +1,6 @@ CREATE TABLE public.line_type_input ( - uuid uuid NOT NULL, + uuid BINARY(16) NOT NULL, id varchar NOT NULL, v_rated double precision NOT NULL, i_max double precision NOT NULL, diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index fd2d36637..dd7c536aa 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -25,7 +25,7 @@ - + From b4a04ae3bf1e2cfb7084b385dfa8bd32b0d651ab Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Wed, 4 Oct 2023 11:23:46 +0200 Subject: [PATCH 02/19] New quoting --- .../edu/ie3/datamodel/io/sink/SqlSink.java | 46 +++++++------------ .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 2 +- .../datamodel/io/sink/_sql/load_profile.sql | 4 +- .../datamodel/io/sink/_sql/time_series.sql | 10 ++-- 4 files changed, 25 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index e1c1c4e0b..ae357217e 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -1,5 +1,6 @@ package edu.ie3.datamodel.io.sink; +import com.couchbase.client.core.deps.io.perfmark.Link; import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.DatabaseIdentifier; import edu.ie3.datamodel.io.connectors.SqlConnector; @@ -207,7 +208,7 @@ private , V extends Value> */ private void insertListIgnoreNested(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { try { - String[] headerElements = StringUtils.camelCaseToSnakeCase(processorProvider.getHeaderElements(cls)); + String[] headerElements = processorProvider.getHeaderElements(cls); String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); connector.executeUpdate(query); @@ -248,7 +249,6 @@ private , V extends Value> void persistTimeSeries( Set> entityFieldData = processorProvider.handleTimeSeries(timeSeries); query = query + entityFieldData.stream().map( - //data -> createTimeSeriesQueryLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", "ON CONFLICT (uuid) DO NOTHING;")); data -> queryTimeSeriesValueLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", ";")); try { connector.executeUpdate(query); @@ -348,7 +348,7 @@ public void persistJointGrid(JointGridContainer jointGridContainer) throws SQLEx } private void insert(C entity, DatabaseIdentifier identifier) throws SQLException { try { - String[] headerElements = StringUtils.camelCaseToSnakeCase(processorProvider.getHeaderElements(entity.getClass())); + String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(entity.getClass()).orElseThrow(), headerElements) + queryValueLine(entity, headerElements, identifier) + ";"; connector.executeUpdate(query); } catch (ProcessorProviderException e) { @@ -369,7 +369,9 @@ private String createInsertQueryBodyIgnoreConflict(List .handleEntities(entities); String queryBody = ""; queryBody = queryBody + entityFieldData.stream().map( - data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)).collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + data -> + queryValueLine(sqlEntityFieldData(data), headerElements, identifier) + ).collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); return queryBody; } @@ -411,30 +413,16 @@ private String queryTimeSeriesValueLine(Map entityFieldData, Str return writeOneLine(Stream.concat(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery()), Stream.of(quote(TSuuid,"'")))); } - /** - * Quote a given entity field - */ + private LinkedHashMap sqlEntityFieldData( - LinkedHashMap entityFieldData) { - - return entityFieldData.entrySet().stream() - .map( - mapEntry -> - new AbstractMap.SimpleEntry<>( - mapEntry.getKey(), - quote(mapEntry.getValue(), "'"))) - .collect( - Collectors.toMap( - AbstractMap.SimpleEntry::getKey, - AbstractMap.SimpleEntry::getValue, - (v1, v2) -> { - throw new IllegalStateException( - "Converting entity data to RFC 4180 compliant strings has lead to duplicate keys. Initial input:\n\t" - + entityFieldData.entrySet().stream() - .map(entry -> entry.getKey() + " = " + entry.getValue()) - .collect(Collectors.joining(",\n\t"))); - }, - LinkedHashMap::new)); + LinkedHashMap entityFieldData + ) { + LinkedHashMap quotedEntityFieldData = new LinkedHashMap<>(entityFieldData); + quotedEntityFieldData.replaceAll( + (key, ent) -> quote(ent, "'") + ); + + return quotedEntityFieldData; } @@ -450,7 +438,7 @@ private static String basicInsertQuery(String schemaName, String tableName) { */ private String basicInsertQueryValuesGrid(String schemaName, String tableName, String[] headerElements) { String[] addParams = {GRID_NAME, GRID_UUID}; - return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(headerElements, addParams) + "\nVALUES\n"; + return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; } /** @@ -458,7 +446,7 @@ private String basicInsertQueryValuesGrid(String schemaName, String tableName, S */ private String basicInsertQueryValuesITS(String schemaName, String tableName, String[] headerElements) { String[] addParams = {GRID_NAME, GRID_UUID, TIME_SERIES}; - return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(headerElements, addParams) + "\nVALUES\n"; + return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index b8759a546..7b4878461 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -293,7 +293,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri then: def exception = thrown(SQLException) - exception.message == "ERROR: invalid input syntax for type uuid: \"null\"\n Position: 365" + exception.message.contains("ERROR: invalid input syntax for type uuid: \"null\"\n") cleanup: sink.shutdown() diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql index 45cf68173..13ff7f12c 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql @@ -2,8 +2,8 @@ CREATE TABLE public.load_profile_g2 ( uuid uuid PRIMARY KEY, time_series uuid NOT NULL, - dayOfWeek TEXT NOT NULL, - quarterHourOfDay TEXT NOT NULL, + day_of_week TEXT NOT NULL, + quarter_hour_of_day TEXT NOT NULL, p double precision, grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql index 23184d899..640be69d0 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql @@ -55,7 +55,7 @@ CREATE TABLE public.time_series_h uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, - heatDemand double precision, + heat_demand double precision, grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) @@ -72,7 +72,7 @@ CREATE TABLE public.time_series_ph time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, - heatDemand double precision, + heat_demand double precision, grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) @@ -90,7 +90,7 @@ CREATE TABLE public.time_series_pqh time timestamp with time zone NOT NULL, p double precision, q double precision, - heatDemand double precision, + heat_demand double precision, grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) @@ -107,8 +107,8 @@ CREATE TABLE public.time_series_weather time_series uuid NOT NULL, coordinate TEXT NOT NULL, time timestamp with time zone NOT NULL, - diffuseIrradiance double precision, - directIrradiance double precision, + diffuse_irradiance double precision, + direct_irradiance double precision, direction double precision, temperature double precision, velocity double precision, From d5f1c9f9ed755fda95a702625e9fb73465b37e43 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Thu, 12 Oct 2023 15:37:32 +0200 Subject: [PATCH 03/19] documentations --- .../edu/ie3/datamodel/io/sink/SqlSink.java | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index ae357217e..a8bc24480 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -1,12 +1,12 @@ package edu.ie3.datamodel.io.sink; -import com.couchbase.client.core.deps.io.perfmark.Link; import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.DatabaseIdentifier; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.extractor.Extractor; import edu.ie3.datamodel.io.extractor.NestedEntity; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; +import edu.ie3.datamodel.io.processor.EntityProcessor; import edu.ie3.datamodel.io.processor.ProcessorProvider; import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey; import edu.ie3.datamodel.models.UniqueEntity; @@ -78,23 +78,32 @@ public void shutdown() { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + /** + * Entry point of a data sink to persist multiple entities in a collection. + * @param entities a collection of entities that should be persisted + * @param identifier identifier of the grid + * @param bounded to be all unique entities. Handling of specific entities is normally then + * executed by a specific {@link EntityProcessor} + * @throws SQLException + */ public void persistAll(Collection entities, DatabaseIdentifier identifier) throws SQLException { // Extract nested entities and add them to the set of entities Set entitiesToAdd = new HashSet<>(entities); // entities to persist - entities.stream().filter( - entity -> entity instanceof NestedEntity - ).forEach( + entities.stream().forEach( entity -> { - try { - entitiesToAdd.addAll((List) Extractor.extractElements((NestedEntity) entity).stream().toList()); - } catch (ExtractorException e) { - log.error( - String.format( - "An error occurred during extraction of nested entity'%s': ", - entity.getClass()), - e); + if (entity instanceof NestedEntity nestedEntity) { + try { + entitiesToAdd.addAll((List) Extractor.extractElements(nestedEntity).stream().toList()); + } catch (ExtractorException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + entity.getClass()), + e); + } } - }); + } + ); // Persist the entities in hierarchic order to avoid failure because of foreign keys for (Class cls : hierarchicInsert()) { @@ -107,10 +116,21 @@ public void persistAll(Collection entities, Database persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities } - public void persistAllIgnoreNested(Collection entities, DatabaseIdentifier identifier) { + /** + * Persist multiple input entities in a collection. In contrast to {@link SqlSink#persistAll} this function does not extract nested entities. + * @param entities a collection of entities that should be persisted + * @param identifier identifier of the grid + */ + public void persistAllIgnoreNested(Collection entities, DatabaseIdentifier identifier) { persistMixedList(new ArrayList<>(entities), identifier); } + /** + * Persist an entity. By default this method take care about the extraction process of nested entities (if any) + * @param entity the entity that should be persisted + * @param identifier identifier of the grid + * @throws SQLException + */ public void persist(C entity, DatabaseIdentifier identifier) throws SQLException { if (entity instanceof InputEntity inputEntity) { persistIncludeNested(inputEntity, identifier); @@ -123,6 +143,12 @@ public void persist(C entity, DatabaseIdentifier identi } } + /** + * Persist an entity. In contrast to {@link SqlSink#persist} this function doe not extract nested entities. + * @param entity the entity that should be persisted + * @param identifier identifier of the grid + * @throws SQLException + */ public void persistIgnoreNested(C entity, DatabaseIdentifier identifier) throws SQLException { insert(entity, identifier); } From 31ff1c880c82acc915d61a2c9fabd1860b0f41cb Mon Sep 17 00:00:00 2001 From: smjobaoo Date: Tue, 21 Nov 2023 16:46:42 +0100 Subject: [PATCH 04/19] implementend createTable --- .../ie3/datamodel/io/DatabaseIdentifier.java | 43 ------ .../edu/ie3/datamodel/io/DbGridMetadata.java | 24 ++++ .../java/edu/ie3/datamodel/io/SqlUtils.java | 53 ++++--- .../io/naming/DatabaseNamingStrategy.java | 2 +- .../edu/ie3/datamodel/io/sink/SqlSink.java | 136 ++++++++---------- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 95 +++++------- .../edu/ie3/datamodel/io/sink/_sql/grids.sql | 13 ++ .../datamodel/io/sink/_sql/input_entities.sql | 42 ++---- .../datamodel/io/sink/_sql/load_profile.sql | 3 +- .../io/sink/_sql/result_entities.sql | 6 - .../datamodel/io/sink/_sql/time_series.sql | 21 +-- .../edu/ie3/datamodel/io/sink/_sql/types.sql | 11 +- 12 files changed, 182 insertions(+), 267 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java create mode 100644 src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java create mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql diff --git a/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java b/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java deleted file mode 100644 index 14d9c0738..000000000 --- a/src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java +++ /dev/null @@ -1,43 +0,0 @@ -package edu.ie3.datamodel.io; - -import java.util.UUID; -import java.util.stream.Stream; - -import static edu.ie3.datamodel.io.IoUtil.quote; - -/** - * Class for identification of grids and results in SQL databases. - */ -public class DatabaseIdentifier { - - private final String identifier; - private final UUID uuid; - - public DatabaseIdentifier( - String identifier, - UUID uuid - ) { - this.identifier = identifier; - this.uuid = uuid; - } - - public String getIdentifier() { - return identifier; - } - - public UUID getUuid() { - return uuid; - } - - public String toString() { - return "identifier=" + identifier + ", uuid=" + uuid.toString(); - } - - public String[] getQueryString() { - return new String[]{quote(identifier, "'"), quote(uuid.toString(), "'")}; - } - - public Stream getStreamForQuery() { - return Stream.concat(Stream.of(quote(identifier, "'")), Stream.of(quote(uuid.toString(), "'"))); - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java new file mode 100644 index 000000000..ae1e2b6cb --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java @@ -0,0 +1,24 @@ +package edu.ie3.datamodel.io; + +import java.util.UUID; +import java.util.stream.Stream; + +import static edu.ie3.datamodel.io.IoUtil.quote; + +/** + * Class for identification of entities and results from grids in SQL databases. + */ +public record DbGridMetadata(String gridName, UUID uuid) { + + public static final String GRID_NAME = "grid_name"; + public static final String GRID_UUID = "grid_uuid"; + + public String toString() { + return GRID_NAME + "=" + gridName + ", " + GRID_UUID + "=" + uuid.toString(); + } + + public Stream getStreamForQuery() { + return Stream.of(quote(uuid.toString(), "'")); + } + +} diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index c0fe83645..458dbcc6e 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -4,17 +4,22 @@ import edu.ie3.datamodel.exceptions.ProcessorProviderException; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.processor.ProcessorProvider; +import edu.ie3.datamodel.io.sink.SqlSink; import edu.ie3.datamodel.models.UniqueEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; public class SqlUtils { + protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; private SqlUtils() { @@ -43,31 +48,29 @@ private static String beginQueryCreateTable( return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; } - public static String getDataTypes(Class cls) { - try { - ProcessorProvider processorProvider = new ProcessorProvider(); - DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); - String body = ""; - Stream dataTypes; - String[] headerElements = processorProvider.getHeaderElements(cls); - Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_name", "grid_uuid")) ; - Stream dtHeader = strHeader.map( - element -> { - return camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element)); - } - ); - return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" + - "WITHOUT OIDS\n" + - "\t" + "TABLESPACE pg_default;\n"; - } catch (EntityProcessorException e) { - return ""; - } catch (ProcessorProviderException e) { - return ""; - } + public static String queryForGridTable( + String schemaName, + String tableName + ) { + return beginQueryCreateTable(schemaName, tableName) + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" + endQueryCreateTable; + } + + public static String getDataTypes(Class cls) throws EntityProcessorException, ProcessorProviderException { + ProcessorProvider processorProvider = new ProcessorProvider(); + DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); + String[] headerElements = processorProvider.getHeaderElements(cls); + Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_uuid")) ; + Stream dtHeader = strHeader.map( + element -> camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element)) + ); + return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" + + "WITHOUT OIDS\n" + + "\t" + "TABLESPACE pg_default;\n"; } public static Map classToDataType() { HashMap map = new HashMap(); + map.put("uuid", "uuid PRIMARY KEY"); map.put("time_series", "uuid NOT NULL"); map.put("time", "timestamp with time zone NOT NULL"); @@ -81,9 +84,7 @@ public static Map classToDataType() { map.put("id", "TEXT NOT NULL"); map.put("market_reaction", "bool NOT NULL"); map.put("node", "uuid NOT NULL"); - //map.put("operatesFrom", "timestamp with time zone"); map.put("operates_from", "timestamp with time zone"); - //map.put("operatesUntil", "timestamp with time zone"); map.put("operates_until", "timestamp with time zone"); map.put("operator", "uuid"); map.put("q_characteristics", "TEXT NOT NULL"); @@ -94,7 +95,6 @@ public static Map classToDataType() { map.put("type", "uuid NOT NULL"); //EVCS map.put("olm_characteristic", "TEXT NOT NULL"); map.put("parallel_devices", "int NOT NULL"); - //map.put("parallelDevices", "int NOT NULL"); map.put("cos_phi_rated", "TEXT NOT NULL"); map.put("dsm", "bool NOT NULL"); map.put("e_cons_annual", "double precision NOT NULL"); @@ -108,15 +108,12 @@ public static Map classToDataType() { map.put("v_mag", "bool NOT NULL"); map.put("slack", "bool NOT NULL"); map.put("subnet", "int NOT NULL"); - //map.put("vRated", "double precision NOT NULL"); map.put("v_rated", "double precision NOT NULL"); map.put("v_target", "double precision NOT NULL"); - //map.put("vTarget", "double precision NOT NULL"); map.put("volt_lvl", "TEXT NOT NULL"); map.put("charging_points", "int NOT NULL"); map.put("location_type", "TEXT NOT NULL"); map.put("v_2g_support", "bool NOT NULL"); - //map.put("voltLvl", "TEXT NOT NULL"); map.put("albedo", "double precision NOT NULL"); map.put("azimuth", "double precision NOT NULL"); @@ -179,6 +176,8 @@ public static Map classToDataType() { map.put("opex", "double precision NOT NULL"); map.put("active_power_gradient", "double precision NOT NULL"); + // not all data types are implemented + return map; } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java index 1801ee345..482815fa5 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java @@ -73,7 +73,7 @@ public Optional getEntityName(Class cls) { /** * Provides the name of a time series. Used to determine the table name in SQL database. - * @param timeSeries + * @param timeSeries to be named TimeSeries * @return the table name */ public , E extends TimeSeriesEntry, V extends Value> diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index a8bc24480..ddfa4d8a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -1,7 +1,7 @@ package edu.ie3.datamodel.io.sink; import edu.ie3.datamodel.exceptions.*; -import edu.ie3.datamodel.io.DatabaseIdentifier; +import edu.ie3.datamodel.io.DbGridMetadata; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.extractor.Extractor; import edu.ie3.datamodel.io.extractor.NestedEntity; @@ -24,6 +24,7 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; + import edu.ie3.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +35,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static edu.ie3.datamodel.io.SqlUtils.quote; -import static java.util.UUID.randomUUID; +import static edu.ie3.datamodel.io.SqlUtils.*; import static java.util.stream.Collectors.groupingBy; @@ -48,8 +48,6 @@ public class SqlSink { private final ProcessorProvider processorProvider; private final String schemaName; - private static final String GRID_NAME = "grid_name"; - private static final String GRID_UUID = "grid_uuid"; private static final String TIME_SERIES = "time_series"; public SqlSink( @@ -86,7 +84,7 @@ public void shutdown() { * executed by a specific {@link EntityProcessor} * @throws SQLException */ - public void persistAll(Collection entities, DatabaseIdentifier identifier) throws SQLException { + public void persistAll(Collection entities, DbGridMetadata identifier) throws SQLException { // Extract nested entities and add them to the set of entities Set entitiesToAdd = new HashSet<>(entities); // entities to persist entities.stream().forEach( @@ -121,7 +119,7 @@ public void persistAll(Collection entities, Database * @param entities a collection of entities that should be persisted * @param identifier identifier of the grid */ - public void persistAllIgnoreNested(Collection entities, DatabaseIdentifier identifier) { + public void persistAllIgnoreNested(Collection entities, DbGridMetadata identifier) { persistMixedList(new ArrayList<>(entities), identifier); } @@ -131,7 +129,7 @@ public void persistAllIgnoreNested(Collection entitie * @param identifier identifier of the grid * @throws SQLException */ - public void persist(C entity, DatabaseIdentifier identifier) throws SQLException { + public void persist(C entity, DbGridMetadata identifier) throws SQLException { if (entity instanceof InputEntity inputEntity) { persistIncludeNested(inputEntity, identifier); } else if (entity instanceof ResultEntity resultEntity) { @@ -144,18 +142,18 @@ public void persist(C entity, DatabaseIdentifier identi } /** - * Persist an entity. In contrast to {@link SqlSink#persist} this function doe not extract nested entities. + * Persist an entity. In contrast to {@link SqlSink#persist} this function does not extract nested entities. * @param entity the entity that should be persisted * @param identifier identifier of the grid * @throws SQLException */ - public void persistIgnoreNested(C entity, DatabaseIdentifier identifier) throws SQLException { + public void persistIgnoreNested(C entity, DbGridMetadata identifier) throws SQLException { insert(entity, identifier); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - protected void persistListIncludeNested(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + protected void persistListIncludeNested(List entities, Class cls, DbGridMetadata identifier) throws SQLException { if (NestedEntity.class.isAssignableFrom(cls)) { entities.forEach( entity -> { @@ -177,13 +175,13 @@ protected void persistListIncludeNested(List entitie } } - public void persistIncludeNested(C entity, DatabaseIdentifier identifier) throws SQLException { + public void persistIncludeNested(C entity, DbGridMetadata identifier) throws SQLException { Set entitiesToAdd = new HashSet<>(); entitiesToAdd.add(entity); persistAll(entitiesToAdd, identifier); } - private void persistMixedList(List entities, DatabaseIdentifier identifier) { + private void persistMixedList(List entities, DbGridMetadata identifier) { Map, List> entitiesPerClass = entities.stream() .collect(groupingBy(entity -> (Class) entity.getClass())); entitiesPerClass.forEach( @@ -191,10 +189,6 @@ private void persistMixedList(List entities, Databas try { persistList(ent, cls, identifier); } catch (SQLException e) { - log.error(String.format( - "An error occurred during extraction of entity'%s': ", - cls.getSimpleName()), - e); throw new RuntimeException(String.format( "An error occurred during extraction of entity '%s', SQLReason: '%s'", cls.getSimpleName(), e.getMessage()), e); @@ -203,16 +197,10 @@ private void persistMixedList(List entities, Databas ); } - private , V extends Value> void persistList(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + private , V extends Value> void persistList(List entities, Class cls, DbGridMetadata identifier) throws SQLException { // Check if there are only elements of the same class Class firstClass = entities.get(0).getClass(); - boolean allSameClass = true; - for (Object obj : entities) { - if (obj.getClass() != firstClass) { - allSameClass = false; - break; - } - } + boolean allSameClass = entities.stream().allMatch(e -> e.getClass() == firstClass); if (allSameClass) { if (InputEntity.class.isAssignableFrom(cls)) { @@ -220,7 +208,15 @@ private , V extends Value> } else if (ResultEntity.class.isAssignableFrom(cls)) { insertListIgnoreNested(entities, cls, identifier); } else if (TimeSeries.class.isAssignableFrom(cls)) { - persistListOfTimeSeries((List>) entities, identifier); + entities.forEach(ts -> { + try { + persistTimeSeries((TimeSeries) ts, identifier); + } catch (SQLException e) { + throw new RuntimeException(String.format( + "An error occurred during extraction of entity '%s', SQLReason: '%s'", + cls.getSimpleName(), e.getMessage()), e); + } + }); } else { log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); } @@ -232,30 +228,18 @@ private , V extends Value> /** * Writes a list of entities into a sql table. It's necessary that all entities have the same class. */ - private void insertListIgnoreNested(List entities, Class cls, DatabaseIdentifier identifier) throws SQLException { + private void insertListIgnoreNested(List entities, Class cls, DbGridMetadata identifier) throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(cls); String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); connector.executeUpdate(query); - //int expectedElements = entities.size(); - //int addedElements = connector.executeUpdate(query); - //System.out.println("Insert " + cls.getSimpleName() + ": " + " expected " + expectedElements + ", added " + addedElements); } catch (ProcessorProviderException e) { log.error("Exception occurred during processor request: ", e); } } - private , V extends Value> void persistListOfTimeSeries( - List> list, DatabaseIdentifier identifier - ) throws SQLException { - for (TimeSeries ts : list) { - persistTimeSeries(ts, identifier); - } - } - - - protected , V extends Value> void persistTimeSeries(TimeSeries timeSeries, DatabaseIdentifier identifier) throws SQLException { + protected , V extends Value> void persistTimeSeries(TimeSeries timeSeries, DbGridMetadata identifier) throws SQLException { try { TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); String[] headerElements = processorProvider.getHeaderElements(key); @@ -269,7 +253,7 @@ protected , V extends Value> void persistTimeSeries } private , V extends Value> void persistTimeSeries( - TimeSeries timeSeries, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException, IOException, SQLException { + TimeSeries timeSeries, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException, IOException, SQLException { try { String query = basicInsertQueryValuesITS(schemaName, databaseNamingStrategy.getEntityName(timeSeries).orElseThrow(), headerElements); Set> entityFieldData = @@ -278,20 +262,19 @@ private , V extends Value> void persistTimeSeries( data -> queryTimeSeriesValueLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", ";")); try { connector.executeUpdate(query); - //int expectedElements = timeSeries.getEntries().size(); - //int addedElements = connector.executeUpdate(query); - //System.out.println("Insert " + "TimeSeries" + ": " + " expected " + expectedElements + ", added " + addedElements); } catch (SQLException e) { - throw new SQLException(String.format("Error at execution of query \"%1.127s\": ", query), e); + throw new RuntimeException(String.format( + "An error occurred during extraction of the time series, SQLReason: '%s'", + e.getMessage()), e); } } catch (ProcessorProviderException e) { throw new ProcessorProviderException("Exception occurred during processor request: ", e); } } - public void persistJointGrid(JointGridContainer jointGridContainer) throws SQLException { - DatabaseIdentifier identifier = new DatabaseIdentifier( - jointGridContainer.getGridName(), randomUUID() + public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) throws SQLException { + DbGridMetadata identifier = new DbGridMetadata( + jointGridContainer.getGridName(), gridUUID ); // get raw grid entities with types or operators @@ -372,7 +355,7 @@ public void persistJointGrid(JointGridContainer jointGridContainer) throws SQLEx // persist all entities persistAll(toAdd, identifier); } - private void insert(C entity, DatabaseIdentifier identifier) throws SQLException { + private void insert(C entity, DbGridMetadata identifier) throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(entity.getClass()).orElseThrow(), headerElements) + queryValueLine(entity, headerElements, identifier) + ";"; @@ -385,11 +368,10 @@ private void insert(C entity, DatabaseIdentifier identi // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- /** - * Provides the value lists for an insertion query. Conflicts because of the primary key uuid will be ignored. - * + * Provides the value lists for an insertion query. Conflicts because of the primary key 'uuid' will be ignored. Conflicts can occur if an entity (e.g. node) already exist. * WARNING: It's assumed that all entities are from the same class C. */ - private String createInsertQueryBodyIgnoreConflict(List entities, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { + private String createInsertQueryBodyIgnoreConflict(List entities, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { Set> entityFieldData = processorProvider .handleEntities(entities); @@ -406,7 +388,7 @@ private String createInsertQueryBodyIgnoreConflict(List * * WARNING: It's assumed that all entities are from the same class C. */ - private String createInsertQueryBody(List entities, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { + private String createInsertQueryBody(List entities, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { Set> entityFieldData = processorProvider .handleEntities(entities); @@ -419,27 +401,21 @@ private String createInsertQueryBody(List entities, /** * Creates a line with the values of one entity for an insertion query using the entityFieldData. */ - private String queryValueLine(LinkedHashMap entityFieldData, String[] headerElements, DatabaseIdentifier identifier) { + private String queryValueLine(LinkedHashMap entityFieldData, String[] headerElements, DbGridMetadata identifier) { return writeOneLine(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery())); } /** * Creates a line with the values of one entity for an insertion query. */ - private String queryValueLine(C entity, String[] headerElements, DatabaseIdentifier identifier) throws ProcessorProviderException { - LinkedHashMap entityFieldData = - processorProvider - .handleEntity(entity) - .map(this::sqlEntityFieldData) - .getOrThrow(); - return writeOneLine(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery())); + private String queryValueLine(C entity, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { + return queryValueLine(processorProvider.handleEntity(entity).map(this::sqlEntityFieldData).getOrThrow(), headerElements, identifier); } - private String queryTimeSeriesValueLine(Map entityFieldData, String[] headerElements, DatabaseIdentifier identifier, String TSuuid) { + private String queryTimeSeriesValueLine(Map entityFieldData, String[] headerElements, DbGridMetadata identifier, String TSuuid) { return writeOneLine(Stream.concat(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery()), Stream.of(quote(TSuuid,"'")))); } - private LinkedHashMap sqlEntityFieldData( LinkedHashMap entityFieldData ) { @@ -451,7 +427,6 @@ private LinkedHashMap sqlEntityFieldData( return quotedEntityFieldData; } - /** * "INSERT INTO" line with schemaName.tableName */ @@ -463,7 +438,7 @@ private static String basicInsertQuery(String schemaName, String tableName) { * Provides the insert, column names, grid identifier and the VALUES statement for a query. */ private String basicInsertQueryValuesGrid(String schemaName, String tableName, String[] headerElements) { - String[] addParams = {GRID_NAME, GRID_UUID}; + String[] addParams = {DbGridMetadata.GRID_UUID}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; } @@ -471,7 +446,9 @@ private String basicInsertQueryValuesGrid(String schemaName, String tableName, S * Provides the insert, column names, grid identifier, time_series uuid and the VALUES statement for a query. */ private String basicInsertQueryValuesITS(String schemaName, String tableName, String[] headerElements) { - String[] addParams = {GRID_NAME, GRID_UUID, TIME_SERIES}; + String[] addParams = { + DbGridMetadata.GRID_UUID, + TIME_SERIES}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; } @@ -489,6 +466,21 @@ private String writeOneLine(String[] entries, String[] addParams) { return writeOneLine(Stream.concat(Arrays.stream(entries), Arrays.stream(addParams))); } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + public void createClassTable( + Class cls + ) throws SQLException, ProcessorProviderException, EntityProcessorException { + String query = getDataTypes(cls); + connector.executeUpdate(query); + } + + public void createGridTable( + String schemaName, String tableName + ) throws SQLException { + String query = queryForGridTable(schemaName, tableName); + connector.executeUpdate(query); + } /** * @return insertion order for unique entities @@ -506,16 +498,4 @@ private static List> hierarchicInsert() { //8. Rest return sortedInsert; } - - private String entitiesToString(Collection entities) { - String res = "[\n"; - int no = 0; - Iterator iterator = entities.iterator(); - while (iterator.hasNext()) { - res = res + "\t (" + no + ") " + iterator.next().toString() + "\n"; - no++; - } - res = res + "]"; - return res; - } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index 7b4878461..cba22bd12 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -1,21 +1,14 @@ package edu.ie3.datamodel.io.sink -import edu.ie3.datamodel.io.DatabaseIdentifier -import edu.ie3.datamodel.io.SqlUtils +import edu.ie3.datamodel.io.DbGridMetadata import edu.ie3.datamodel.io.connectors.SqlConnector -import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy -import edu.ie3.datamodel.io.naming.FileNamingStrategy -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.processor.ProcessorProvider import edu.ie3.datamodel.io.processor.input.InputEntityProcessor import edu.ie3.datamodel.io.processor.result.ResultEntityProcessor import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessor import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey -import edu.ie3.datamodel.io.source.csv.CsvTimeSeriesSource import edu.ie3.datamodel.io.source.sql.SqlDataSource -import edu.ie3.datamodel.io.source.sql.SqlTimeSeriesSource import edu.ie3.datamodel.models.OperationTime import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.input.NodeInput @@ -30,10 +23,7 @@ import edu.ie3.datamodel.models.input.system.EmInput import edu.ie3.datamodel.models.input.system.EvcsInput import edu.ie3.datamodel.models.input.system.LoadInput import edu.ie3.datamodel.models.input.system.PvInput -import edu.ie3.datamodel.models.input.system.StorageInput -import edu.ie3.datamodel.models.input.system.WecInput import edu.ie3.datamodel.models.input.system.characteristic.CosPhiFixed -import edu.ie3.datamodel.models.input.system.type.StorageTypeInput import edu.ie3.datamodel.models.input.thermal.CylindricalStorageInput import edu.ie3.datamodel.models.input.thermal.ThermalBusInput import edu.ie3.datamodel.models.input.thermal.ThermalHouseInput @@ -47,7 +37,6 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput import edu.ie3.datamodel.models.value.EnergyPriceValue import edu.ie3.datamodel.models.value.Value import edu.ie3.test.common.GridTestData @@ -56,8 +45,6 @@ import edu.ie3.test.common.ThermalUnitInputTestData import edu.ie3.test.common.TimeSeriesTestData import edu.ie3.test.helper.TestContainerHelper import edu.ie3.util.TimeUtil -import edu.ie3.util.io.FileIOUtils -import org.jetbrains.annotations.NotNull import org.testcontainers.containers.Container import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.utility.MountableFile @@ -68,17 +55,11 @@ import tech.units.indriya.quantity.Quantities import javax.measure.Quantity import javax.measure.quantity.Power -import java.nio.file.Path import edu.ie3.test.common.SystemParticipantTestData import java.sql.SQLException -import java.time.ZoneId -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import static edu.ie3.datamodel.models.StandardUnits.ENERGY_PRICE -import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM import static edu.ie3.util.quantities.PowerSystemUnits.KILOVOLTAMPERE import static tech.units.indriya.unit.Units.PERCENT @@ -99,7 +80,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri DatabaseNamingStrategy namingStrategy @Shared - DatabaseIdentifier identifier + DbGridMetadata identifier static String schemaName = "public" @@ -113,7 +94,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri namingStrategy = new DatabaseNamingStrategy() - identifier = new DatabaseIdentifier("vn_simona", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120002")) + identifier = new DbGridMetadata("vn_simona", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120002")) sqlSource = new SqlDataSource(connector, schemaName, namingStrategy) } @@ -121,6 +102,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri def setup() { // Execute import script Iterable importFiles = Arrays.asList( + "grids.sql", "types.sql", "result_entities.sql", "input_entities.sql", @@ -336,13 +318,41 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri sink.shutdown() } + def "A valid SqlSink can create a table for class."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + def wec = SystemParticipantTestData.wecInput + + when: + sink.createClassTable(wec.getClass()) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_input", ps -> {}).count() == 0 + } + + def "A valid SqlSink throws an exception if a grid does not exist."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + def failIdentifier = new DbGridMetadata("fail_grid", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120003")) + + sink.persist(individualEnergyPriceTimeSeries, failIdentifier) + + then: + def exception = thrown(RuntimeException) + exception.message.contains("Detail: Key (grid_uuid)=(8e6bd444-4580-11ee-be56-0242ac120003) is not present in table \"grids\".") + + cleanup: + sink.shutdown() + } def "A valid SqlSink should persist a valid joint grid container correctly"() { given: def sink = new SqlSink(schemaName, namingStrategy, connector) when: - sink.persistJointGrid(SampleJointGrid.grid()) + sink.persistJointGrid(SampleJointGrid.grid(), UUID.fromString("297dfac8-83cc-11ee-b962-0242ac120002")) then: sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 6 @@ -360,43 +370,4 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri sink.shutdown() } - /* - def "Ausgabe"() { - given: - System.out.println(SqlUtils.getDataTypes(StorageTypeInput.class)) - //System.out.println(SqlUtils.getDataTypes(StorageInput.class)) - //System.out.println(SqlUtils.getDataTypes(NodeInput.class)) - - System.out.println(SqlUtils.getDataTypes(PvResult.class)) - System.out.println(SqlUtils.getDataTypes(WecResult.class)) - System.out.println(SqlUtils.getDataTypes(EvResult.class)) - System.out.println(SqlUtils.getDataTypes(EvcsResult.class)) - System.out.println(SqlUtils.getDataTypes(EmResult.class)) - System.out.println(SqlUtils.getDataTypes(FlexOptionsResult.class)) - - - System.out.println(SqlUtils.getDataTypes(Transformer2WInput.class)) - System.out.println(SqlUtils.getDataTypes(NodeInput.class)) - System.out.println(SqlUtils.getDataTypes(EvcsInput.class)) - System.out.println(SqlUtils.getDataTypes(LineGraphicInput.class)) - System.out.println(SqlUtils.getDataTypes(NodeGraphicInput.class)) - System.out.println(SqlUtils.getDataTypes(CylindricalStorageInput.class)) - System.out.println(SqlUtils.getDataTypes(ThermalHouseInput.class)) - System.out.println(SqlUtils.getDataTypes(OperatorInput.class)) - System.out.println(SqlUtils.getDataTypes(LineInput.class)) - System.out.println(SqlUtils.getDataTypes(ThermalBusInput.class)) - System.out.println(SqlUtils.getDataTypes(LineTypeInput.class)) - System.out.println(SqlUtils.getDataTypes(LoadInput.class)) - System.out.println(SqlUtils.getDataTypes(EmInput.class)) - - System.out.println(SqlUtils.getDataTypes(Transformer2WTypeInput.class)) - - - when: - def nummer = 1 - - then: - nummer == 1 - } - */ } diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql new file mode 100644 index 000000000..466aad358 --- /dev/null +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql @@ -0,0 +1,13 @@ +CREATE TABLE public.grids +( + uuid uuid PRIMARY KEY, + name TEXT NOT NULL +) + WITHOUT OIDS + TABLESPACE pg_default; + +INSERT INTO + public.grids (uuid, name) +VALUES + ('8e6bd444-4580-11ee-be56-0242ac120002', 'vn_simona'), + ('297dfac8-83cc-11ee-b962-0242ac120002', 'sampleGrid'); \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql index fa6d569aa..1c3bddf01 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql @@ -11,8 +11,7 @@ CREATE TABLE public.node_input v_rated double precision NOT NULL, v_target double precision NOT NULL, volt_lvl TEXT NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -30,8 +29,7 @@ CREATE TABLE public.transformer_2_w_input parallel_devices int NOT NULL, tap_pos int NOT NULL, type uuid NOT NULL REFERENCES transformer_2_w_type_input(uuid), - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -50,8 +48,7 @@ CREATE TABLE public.evcs_input q_characteristics TEXT NOT NULL, type TEXT NOT NULL, v_2g_support bool NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -62,8 +59,7 @@ CREATE TABLE public.line_graphic_input graphic_layer TEXT NOT NULL, line uuid NOT NULL, path TEXT, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -75,8 +71,7 @@ CREATE TABLE public.node_graphic_input node uuid NOT NULL, path TEXT, point TEXT NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -94,8 +89,7 @@ CREATE TABLE public.cylindrical_storage_input storage_volume_lvl double precision NOT NULL, storage_volume_lvl_min double precision NOT NULL, thermal_bus uuid NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -113,8 +107,7 @@ CREATE TABLE public.thermal_house_input target_temperature double precision NOT NULL, thermal_bus uuid NOT NULL, upper_temperature_limit double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -123,8 +116,7 @@ CREATE TABLE public.operator_input ( uuid uuid PRIMARY KEY, id TEXT NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -143,8 +135,7 @@ CREATE TABLE public.line_input operator uuid, parallel_devices int NOT NULL, type uuid NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -156,8 +147,7 @@ CREATE TABLE public.thermal_bus_input operates_from timestamp with time zone, operates_until timestamp with time zone, operator uuid, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -176,8 +166,7 @@ CREATE TABLE public.load_input operator uuid, q_characteristics TEXT NOT NULL, s_rated double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -193,8 +182,7 @@ CREATE TABLE public.em_input operates_until timestamp with time zone, operator uuid, q_characteristics TEXT NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -217,8 +205,7 @@ CREATE TABLE public.pv_input operator uuid, q_characteristics TEXT NOT NULL, s_rated double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -233,8 +220,7 @@ CREATE TABLE public.storage_input operator uuid, q_characteristics TEXT NOT NULL, type uuid NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql index 13ff7f12c..5f3268340 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql @@ -5,8 +5,7 @@ CREATE TABLE public.load_profile_g2 day_of_week TEXT NOT NULL, quarter_hour_of_day TEXT NOT NULL, p double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql index f6ad68d49..924d8efb1 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql @@ -5,7 +5,6 @@ CREATE TABLE public.pv_res p double precision NOT NULL, q double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS @@ -18,7 +17,6 @@ CREATE TABLE public.wec_res p double precision NOT NULL, q double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS @@ -32,7 +30,6 @@ CREATE TABLE public.ev_res q double precision NOT NULL, soc double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS @@ -45,7 +42,6 @@ CREATE TABLE public.evcs_res p double precision NOT NULL, q double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS @@ -58,7 +54,6 @@ CREATE TABLE public.em_res p double precision NOT NULL, q double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS @@ -72,7 +67,6 @@ CREATE TABLE public.flex_options_res p_min double precision NOT NULL, p_ref double precision NOT NULL, time timestamp with time zone NOT NULL, - grid_name TEXT NOT NULL, grid_uuid uuid NOT NULL ) WITHOUT OIDS diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql index 640be69d0..d77ad1895 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql @@ -4,8 +4,7 @@ CREATE TABLE public.time_series_c time_series uuid NOT NULL, time timestamp with time zone NOT NULL, price double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -23,8 +22,7 @@ CREATE TABLE public.time_series_p time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -40,8 +38,7 @@ CREATE TABLE public.time_series_pq time timestamp with time zone NOT NULL, p double precision, q double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -56,8 +53,7 @@ CREATE TABLE public.time_series_h time_series uuid NOT NULL, time timestamp with time zone NOT NULL, heat_demand double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -73,8 +69,7 @@ CREATE TABLE public.time_series_ph time timestamp with time zone NOT NULL, p double precision, heat_demand double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -91,8 +86,7 @@ CREATE TABLE public.time_series_pqh p double precision, q double precision, heat_demand double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -112,8 +106,7 @@ CREATE TABLE public.time_series_weather direction double precision, temperature double precision, velocity double precision, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql index 3d8bd328d..3c4579882 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql @@ -15,8 +15,7 @@ CREATE TABLE public.transformer_2_w_type_input v_rated_a double precision NOT NULL, v_rated_b double precision NOT NULL, x_sc double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -31,8 +30,8 @@ CREATE TABLE public.line_type_input r double precision NOT NULL, v_rated double precision NOT NULL, x double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -52,8 +51,8 @@ CREATE TABLE public.storage_type_input opex double precision NOT NULL, p_max double precision NOT NULL, s_rated double precision NOT NULL, - grid_name TEXT NOT NULL, - grid_uuid uuid NOT NULL + + grid_uuid uuid NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; From 0f46762f3b0702c5b8ba0e03d5a303b69e19582c Mon Sep 17 00:00:00 2001 From: smjobaoo Date: Tue, 21 Nov 2023 16:54:43 +0100 Subject: [PATCH 05/19] spotless --- .../edu/ie3/datamodel/io/DbGridMetadata.java | 30 +- .../java/edu/ie3/datamodel/io/IoUtil.java | 1 - .../java/edu/ie3/datamodel/io/SqlUtils.java | 351 +++---- .../datamodel/io/connectors/SqlConnector.java | 2 +- .../io/naming/DatabaseNamingStrategy.java | 14 +- .../io/processor/ProcessorProvider.java | 3 +- .../edu/ie3/datamodel/io/sink/SqlSink.java | 977 ++++++++++-------- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 586 +++++------ .../ie3/test/common/TimeSeriesTestData.groovy | 4 +- 9 files changed, 1023 insertions(+), 945 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java index ae1e2b6cb..c9bf2fc2f 100644 --- a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java +++ b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java @@ -1,24 +1,26 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ package edu.ie3.datamodel.io; +import static edu.ie3.datamodel.io.IoUtil.quote; + import java.util.UUID; import java.util.stream.Stream; -import static edu.ie3.datamodel.io.IoUtil.quote; - -/** - * Class for identification of entities and results from grids in SQL databases. - */ +/** Class for identification of entities and results from grids in SQL databases. */ public record DbGridMetadata(String gridName, UUID uuid) { - public static final String GRID_NAME = "grid_name"; - public static final String GRID_UUID = "grid_uuid"; - - public String toString() { - return GRID_NAME + "=" + gridName + ", " + GRID_UUID + "=" + uuid.toString(); - } + public static final String GRID_NAME = "grid_name"; + public static final String GRID_UUID = "grid_uuid"; - public Stream getStreamForQuery() { - return Stream.of(quote(uuid.toString(), "'")); - } + public String toString() { + return GRID_NAME + "=" + gridName + ", " + GRID_UUID + "=" + uuid.toString(); + } + public Stream getStreamForQuery() { + return Stream.of(quote(uuid.toString(), "'")); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/IoUtil.java b/src/main/java/edu/ie3/datamodel/io/IoUtil.java index 590ea1b49..9f9f12a74 100644 --- a/src/main/java/edu/ie3/datamodel/io/IoUtil.java +++ b/src/main/java/edu/ie3/datamodel/io/IoUtil.java @@ -63,5 +63,4 @@ public static String quote(String input, String quoteSymbol) { return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; } } - } diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 458dbcc6e..358d3c83c 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -1,191 +1,192 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ package edu.ie3.datamodel.io; +import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; + import edu.ie3.datamodel.exceptions.EntityProcessorException; import edu.ie3.datamodel.exceptions.ProcessorProviderException; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.processor.ProcessorProvider; -import edu.ie3.datamodel.io.sink.SqlSink; import edu.ie3.datamodel.models.UniqueEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SqlUtils { - protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); - private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; - - private SqlUtils() { - throw new IllegalStateException("Utility classes cannot be instantiated"); - } - - - public static String queryForCreation( - String schemaName, - String tableName, - Stream> columnsWithDataTypes - ) { - return beginQueryCreateTable(schemaName, tableName) + - " " - + endQueryCreateTable; - } - - public static String getEndQueryCreateTable() { - return endQueryCreateTable; - } - - private static String beginQueryCreateTable( - String schemaName, - String tableName - ) { - return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; - } - - public static String queryForGridTable( - String schemaName, - String tableName - ) { - return beginQueryCreateTable(schemaName, tableName) + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" + endQueryCreateTable; - } - - public static String getDataTypes(Class cls) throws EntityProcessorException, ProcessorProviderException { - ProcessorProvider processorProvider = new ProcessorProvider(); - DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); - String[] headerElements = processorProvider.getHeaderElements(cls); - Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_uuid")) ; - Stream dtHeader = strHeader.map( - element -> camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element)) - ); - return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" + - "WITHOUT OIDS\n" + - "\t" + "TABLESPACE pg_default;\n"; - } - - public static Map classToDataType() { - HashMap map = new HashMap(); - - map.put("uuid", "uuid PRIMARY KEY"); - map.put("time_series", "uuid NOT NULL"); - map.put("time", "timestamp with time zone NOT NULL"); - map.put("p", "double precision NOT NULL"); - map.put("q", "double precision NOT NULL"); - map.put("c", "double precision NOT NULL"); - map.put("s_rated", "double precision NOT NULL"); - - map.put("cost_controlled", "bool NOT NULL"); - map.put("feed_in_tariff", "int NOT NULL"); - map.put("id", "TEXT NOT NULL"); - map.put("market_reaction", "bool NOT NULL"); - map.put("node", "uuid NOT NULL"); - map.put("operates_from", "timestamp with time zone"); - map.put("operates_until", "timestamp with time zone"); - map.put("operator", "uuid"); - map.put("q_characteristics", "TEXT NOT NULL"); - map.put("geo_position", "TEXT NOT NULL"); - map.put("length", "double precision NOT NULL"); - map.put("node_a", "uuid NOT NULL"); - map.put("node_b", "uuid NOT NULL"); - map.put("type", "uuid NOT NULL"); //EVCS - map.put("olm_characteristic", "TEXT NOT NULL"); - map.put("parallel_devices", "int NOT NULL"); - map.put("cos_phi_rated", "TEXT NOT NULL"); - map.put("dsm", "bool NOT NULL"); - map.put("e_cons_annual", "double precision NOT NULL"); - map.put("load_profile", "TEXT NOT NULL"); - - map.put("auto_tap", "bool NOT NULL"); - map.put("tap_pos", "int NOT NULL"); - map.put("type", "uuid NOT NULL"); - - map.put("v_ang", "bool NOT NULL"); - map.put("v_mag", "bool NOT NULL"); - map.put("slack", "bool NOT NULL"); - map.put("subnet", "int NOT NULL"); - map.put("v_rated", "double precision NOT NULL"); - map.put("v_target", "double precision NOT NULL"); - map.put("volt_lvl", "TEXT NOT NULL"); - map.put("charging_points", "int NOT NULL"); - map.put("location_type", "TEXT NOT NULL"); - map.put("v_2g_support", "bool NOT NULL"); - - map.put("albedo", "double precision NOT NULL"); - map.put("azimuth", "double precision NOT NULL"); - map.put("elevation_angle", "double precision NOT NULL"); - map.put("eta_conv", "double precision NOT NULL"); - map.put("k_g", "double precision NOT NULL"); - map.put("k_t", "double precision NOT NULL"); - - map.put("grid_name", "TEXT NOT NULL"); - map.put("grid_uuid", "uuid NOT NULL"); - - - map.put("b_m", "double precision NOT NULL"); - map.put("d_phi", "double precision NOT NULL"); - map.put("d_v", "double precision NOT NULL"); - map.put("g_m", "double precision NOT NULL"); - map.put("r_sc", "double precision NOT NULL"); - map.put("tap_max", "int NOT NULL"); - map.put("tap_min", "int NOT NULL"); - map.put("tap_neutr", "int NOT NULL"); - map.put("tap_side", "bool NOT NULL"); - map.put("v_rated_a", "int NOT NULL"); - map.put("v_rated_b", "int NOT NULL"); - map.put("x_sc", "int NOT NULL"); - map.put("graphic_layer", "TEXT NOT NULL"); - map.put("line", "uuid NOT NULL"); - map.put("path", "TEXT NOT NULL"); - map.put("point", "TEXT NOT NULL"); - map.put("inlet_temp", "double precision NOT NULL"); - map.put("return_temp", "double precision NOT NULL"); - map.put("storage_volume_lvl", "double precision NOT NULL"); - map.put("storage_volume_lvl_min", "double precision NOT NULL"); - map.put("thermal_bus", "uuid NOT NULL"); - map.put("eth_capa", "double precision NOT NULL"); - map.put("eth_losses", "double precision NOT NULL"); - map.put("lower_temperature_limit", "double precision NOT NULL"); - map.put("target_temperature", "double precision NOT NULL"); - map.put("upper_temperature_limit", "double precision NOT NULL"); - map.put("b", "double precision NOT NULL"); - map.put("g", "double precision NOT NULL"); - map.put("i_max", "double precision NOT NULL"); - map.put("r", "double precision NOT NULL"); - map.put("x", "double precision NOT NULL"); - - map.put("connected_assets", "TEXT NOT NULL"); - map.put("capex", "double precision NOT NULL"); - map.put("control_strategy", "TEXT NOT NULL"); - - map.put("input_model", "uuid NOT NULL"); - map.put("soc", "double precision NOT NULL"); - map.put("p_max", "double precision NOT NULL"); - map.put("p_min", "double precision NOT NULL"); - map.put("p_ref", "double precision NOT NULL"); - - map.put("dod", "double precision NOT NULL"); - map.put("e_storage", "double precision NOT NULL"); - map.put("eta", "double precision NOT NULL"); - map.put("life_cycle", "double precision NOT NULL"); - map.put("life_time", "double precision NOT NULL"); - map.put("opex", "double precision NOT NULL"); - map.put("active_power_gradient", "double precision NOT NULL"); - - // not all data types are implemented - - return map; - } - - public static String quote(String input, String quoteSymbol) { - if (input == "") { - return "NULL"; - } else { - return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; - } + protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); + private static final String endQueryCreateTable = + ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; + + private SqlUtils() { + throw new IllegalStateException("Utility classes cannot be instantiated"); + } + + public static String queryForCreation( + String schemaName, String tableName, Stream> columnsWithDataTypes) { + return beginQueryCreateTable(schemaName, tableName) + " " + endQueryCreateTable; + } + + public static String getEndQueryCreateTable() { + return endQueryCreateTable; + } + + private static String beginQueryCreateTable(String schemaName, String tableName) { + return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; + } + + public static String queryForGridTable(String schemaName, String tableName) { + return beginQueryCreateTable(schemaName, tableName) + + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" + + endQueryCreateTable; + } + + public static String getDataTypes(Class cls) + throws EntityProcessorException, ProcessorProviderException { + ProcessorProvider processorProvider = new ProcessorProvider(); + DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); + String[] headerElements = processorProvider.getHeaderElements(cls); + Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_uuid")); + Stream dtHeader = + strHeader.map( + element -> + camelCaseToSnakeCase(element) + + " " + + classToDataType().get(camelCaseToSnakeCase(element))); + return "CREATE TABLE public." + + namingStrategy.getEntityName(cls).orElseThrow() + + "\n(\n\t" + + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + + "\n)\n\t" + + "WITHOUT OIDS\n" + + "\t" + + "TABLESPACE pg_default;\n"; + } + + public static Map classToDataType() { + HashMap map = new HashMap(); + + map.put("uuid", "uuid PRIMARY KEY"); + map.put("time_series", "uuid NOT NULL"); + map.put("time", "timestamp with time zone NOT NULL"); + map.put("p", "double precision NOT NULL"); + map.put("q", "double precision NOT NULL"); + map.put("c", "double precision NOT NULL"); + map.put("s_rated", "double precision NOT NULL"); + + map.put("cost_controlled", "bool NOT NULL"); + map.put("feed_in_tariff", "int NOT NULL"); + map.put("id", "TEXT NOT NULL"); + map.put("market_reaction", "bool NOT NULL"); + map.put("node", "uuid NOT NULL"); + map.put("operates_from", "timestamp with time zone"); + map.put("operates_until", "timestamp with time zone"); + map.put("operator", "uuid"); + map.put("q_characteristics", "TEXT NOT NULL"); + map.put("geo_position", "TEXT NOT NULL"); + map.put("length", "double precision NOT NULL"); + map.put("node_a", "uuid NOT NULL"); + map.put("node_b", "uuid NOT NULL"); + map.put("type", "uuid NOT NULL"); // EVCS + map.put("olm_characteristic", "TEXT NOT NULL"); + map.put("parallel_devices", "int NOT NULL"); + map.put("cos_phi_rated", "TEXT NOT NULL"); + map.put("dsm", "bool NOT NULL"); + map.put("e_cons_annual", "double precision NOT NULL"); + map.put("load_profile", "TEXT NOT NULL"); + + map.put("auto_tap", "bool NOT NULL"); + map.put("tap_pos", "int NOT NULL"); + map.put("type", "uuid NOT NULL"); + + map.put("v_ang", "bool NOT NULL"); + map.put("v_mag", "bool NOT NULL"); + map.put("slack", "bool NOT NULL"); + map.put("subnet", "int NOT NULL"); + map.put("v_rated", "double precision NOT NULL"); + map.put("v_target", "double precision NOT NULL"); + map.put("volt_lvl", "TEXT NOT NULL"); + map.put("charging_points", "int NOT NULL"); + map.put("location_type", "TEXT NOT NULL"); + map.put("v_2g_support", "bool NOT NULL"); + + map.put("albedo", "double precision NOT NULL"); + map.put("azimuth", "double precision NOT NULL"); + map.put("elevation_angle", "double precision NOT NULL"); + map.put("eta_conv", "double precision NOT NULL"); + map.put("k_g", "double precision NOT NULL"); + map.put("k_t", "double precision NOT NULL"); + + map.put("grid_name", "TEXT NOT NULL"); + map.put("grid_uuid", "uuid NOT NULL"); + + map.put("b_m", "double precision NOT NULL"); + map.put("d_phi", "double precision NOT NULL"); + map.put("d_v", "double precision NOT NULL"); + map.put("g_m", "double precision NOT NULL"); + map.put("r_sc", "double precision NOT NULL"); + map.put("tap_max", "int NOT NULL"); + map.put("tap_min", "int NOT NULL"); + map.put("tap_neutr", "int NOT NULL"); + map.put("tap_side", "bool NOT NULL"); + map.put("v_rated_a", "int NOT NULL"); + map.put("v_rated_b", "int NOT NULL"); + map.put("x_sc", "int NOT NULL"); + map.put("graphic_layer", "TEXT NOT NULL"); + map.put("line", "uuid NOT NULL"); + map.put("path", "TEXT NOT NULL"); + map.put("point", "TEXT NOT NULL"); + map.put("inlet_temp", "double precision NOT NULL"); + map.put("return_temp", "double precision NOT NULL"); + map.put("storage_volume_lvl", "double precision NOT NULL"); + map.put("storage_volume_lvl_min", "double precision NOT NULL"); + map.put("thermal_bus", "uuid NOT NULL"); + map.put("eth_capa", "double precision NOT NULL"); + map.put("eth_losses", "double precision NOT NULL"); + map.put("lower_temperature_limit", "double precision NOT NULL"); + map.put("target_temperature", "double precision NOT NULL"); + map.put("upper_temperature_limit", "double precision NOT NULL"); + map.put("b", "double precision NOT NULL"); + map.put("g", "double precision NOT NULL"); + map.put("i_max", "double precision NOT NULL"); + map.put("r", "double precision NOT NULL"); + map.put("x", "double precision NOT NULL"); + + map.put("connected_assets", "TEXT NOT NULL"); + map.put("capex", "double precision NOT NULL"); + map.put("control_strategy", "TEXT NOT NULL"); + + map.put("input_model", "uuid NOT NULL"); + map.put("soc", "double precision NOT NULL"); + map.put("p_max", "double precision NOT NULL"); + map.put("p_min", "double precision NOT NULL"); + map.put("p_ref", "double precision NOT NULL"); + + map.put("dod", "double precision NOT NULL"); + map.put("e_storage", "double precision NOT NULL"); + map.put("eta", "double precision NOT NULL"); + map.put("life_cycle", "double precision NOT NULL"); + map.put("life_time", "double precision NOT NULL"); + map.put("opex", "double precision NOT NULL"); + map.put("active_power_gradient", "double precision NOT NULL"); + + // not all data types are implemented + + return map; + } + + public static String quote(String input, String quoteSymbol) { + if (input == "") { + return "NULL"; + } else { + return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; } + } } diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index b3b5d704f..91cf38dd9 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -65,7 +65,7 @@ public ResultSet executeQuery(Statement stmt, String query) throws SQLException * @return The number of updates or a negative number if the execution failed */ public int executeUpdate(String updateQuery) throws SQLException { - return getConnection().createStatement().executeUpdate(updateQuery); + return getConnection().createStatement().executeUpdate(updateQuery); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java index 482815fa5..7fadfc45a 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java @@ -5,6 +5,8 @@ */ package edu.ie3.datamodel.io.naming; +import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger; + import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.TimeSeries; @@ -12,11 +14,8 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput; import edu.ie3.datamodel.models.value.Value; - import java.util.Optional; -import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger; - /** A naming strategy for database entities */ public class DatabaseNamingStrategy { @@ -55,6 +54,7 @@ public String getTimeSeriesEntityName(ColumnScheme columnScheme) { /** * Provides the name of a load profile given by the load profile key + * * @param lpKey Load profile key * @return the table name */ @@ -64,6 +64,7 @@ private String getLoadProfileEntityName(String lpKey) { /** * Provides the name of a unique entity class. + * * @param cls Class extends UniqueEntity * @return the table name */ @@ -73,11 +74,12 @@ public Optional getEntityName(Class cls) { /** * Provides the name of a time series. Used to determine the table name in SQL database. + * * @param timeSeries to be named TimeSeries * @return the table name */ public , E extends TimeSeriesEntry, V extends Value> - Optional getEntityName(T timeSeries) { + Optional getEntityName(T timeSeries) { if (timeSeries instanceof IndividualTimeSeries individualTimeSeries) { Optional maybeFirstElement = individualTimeSeries.getEntries().stream().findFirst(); if (maybeFirstElement.isPresent()) { @@ -88,9 +90,7 @@ Optional getEntityName(T timeSeries) { return Optional.empty(); } } else if (timeSeries instanceof LoadProfileInput loadProfileInput) { - return Optional.of( - getLoadProfileEntityName(loadProfileInput.getType().getKey()) - ); + return Optional.of(getLoadProfileEntityName(loadProfileInput.getType().getKey())); } else { logger.error("There is no naming strategy defined for {}", timeSeries); return Optional.empty(); diff --git a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java index b3257ed98..567a70189 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java @@ -79,7 +79,8 @@ Try, ProcessorProviderException> handleEntity(T en .transformF(ProcessorProviderException::new)); } - public Set> handleEntities(List entities) throws ProcessorProviderException { + public Set> handleEntities( + List entities) throws ProcessorProviderException { Set setOfEntities = new HashSet<>(entities); Set> setOfMaps = new HashSet<>(); for (T entity : setOfEntities) { diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index ddfa4d8a7..59d95b8ce 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -1,5 +1,13 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ package edu.ie3.datamodel.io.sink; +import static edu.ie3.datamodel.io.SqlUtils.*; +import static java.util.stream.Collectors.groupingBy; + import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.DbGridMetadata; import edu.ie3.datamodel.io.connectors.SqlConnector; @@ -24,478 +32,541 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; - import edu.ie3.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static edu.ie3.datamodel.io.SqlUtils.*; -import static java.util.stream.Collectors.groupingBy; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SqlSink { - protected static final Logger log = LoggerFactory.getLogger(SqlSink.class); - - private final SqlConnector connector; - private final DatabaseNamingStrategy databaseNamingStrategy; - private final ProcessorProvider processorProvider; - private final String schemaName; - - private static final String TIME_SERIES = "time_series"; - - public SqlSink( - String schemaName, - DatabaseNamingStrategy databaseNamingStrategy, - SqlConnector connector - ) throws EntityProcessorException { - this(schemaName, new ProcessorProvider(), databaseNamingStrategy, connector); - } - - public SqlSink( - String schemaName, - ProcessorProvider processorProvider, - DatabaseNamingStrategy databaseNamingStrategy, - SqlConnector connector - ) { - this.connector = connector; - this.databaseNamingStrategy = databaseNamingStrategy; - this.processorProvider = processorProvider; - this.schemaName = schemaName; - } - - public void shutdown() { - connector.shutdown(); - } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - /** - * Entry point of a data sink to persist multiple entities in a collection. - * @param entities a collection of entities that should be persisted - * @param identifier identifier of the grid - * @param bounded to be all unique entities. Handling of specific entities is normally then - * executed by a specific {@link EntityProcessor} - * @throws SQLException - */ - public void persistAll(Collection entities, DbGridMetadata identifier) throws SQLException { - // Extract nested entities and add them to the set of entities - Set entitiesToAdd = new HashSet<>(entities); // entities to persist - entities.stream().forEach( - entity -> { - if (entity instanceof NestedEntity nestedEntity) { - try { - entitiesToAdd.addAll((List) Extractor.extractElements(nestedEntity).stream().toList()); - } catch (ExtractorException e) { - log.error( - String.format( - "An error occurred during extraction of nested entity'%s': ", - entity.getClass()), - e); - } - } + protected static final Logger log = LoggerFactory.getLogger(SqlSink.class); + + private final SqlConnector connector; + private final DatabaseNamingStrategy databaseNamingStrategy; + private final ProcessorProvider processorProvider; + private final String schemaName; + + private static final String TIME_SERIES = "time_series"; + + public SqlSink( + String schemaName, DatabaseNamingStrategy databaseNamingStrategy, SqlConnector connector) + throws EntityProcessorException { + this(schemaName, new ProcessorProvider(), databaseNamingStrategy, connector); + } + + public SqlSink( + String schemaName, + ProcessorProvider processorProvider, + DatabaseNamingStrategy databaseNamingStrategy, + SqlConnector connector) { + this.connector = connector; + this.databaseNamingStrategy = databaseNamingStrategy; + this.processorProvider = processorProvider; + this.schemaName = schemaName; + } + + public void shutdown() { + connector.shutdown(); + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + /** + * Entry point of a data sink to persist multiple entities in a collection. + * + * @param entities a collection of entities that should be persisted + * @param identifier identifier of the grid + * @param bounded to be all unique entities. Handling of specific entities is normally then + * executed by a specific {@link EntityProcessor} + * @throws SQLException + */ + public void persistAll(Collection entities, DbGridMetadata identifier) + throws SQLException { + // Extract nested entities and add them to the set of entities + Set entitiesToAdd = new HashSet<>(entities); // entities to persist + entities.stream() + .forEach( + entity -> { + if (entity instanceof NestedEntity nestedEntity) { + try { + entitiesToAdd.addAll( + (List) Extractor.extractElements(nestedEntity).stream().toList()); + } catch (ExtractorException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + entity.getClass()), + e); } - ); - - // Persist the entities in hierarchic order to avoid failure because of foreign keys - for (Class cls : hierarchicInsert()) { - persistMixedList( - entitiesToAdd.stream().filter( - ent -> cls.isAssignableFrom(ent.getClass()) - ).collect(Collectors.toList()), identifier); - entitiesToAdd.removeIf(ent -> cls.isAssignableFrom(ent.getClass())); // maybe it's not necessary but I'm not sure if there are entities who aren't in the hierarchic structure - } - persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities - } - - /** - * Persist multiple input entities in a collection. In contrast to {@link SqlSink#persistAll} this function does not extract nested entities. - * @param entities a collection of entities that should be persisted - * @param identifier identifier of the grid - */ - public void persistAllIgnoreNested(Collection entities, DbGridMetadata identifier) { - persistMixedList(new ArrayList<>(entities), identifier); - } - - /** - * Persist an entity. By default this method take care about the extraction process of nested entities (if any) - * @param entity the entity that should be persisted - * @param identifier identifier of the grid - * @throws SQLException - */ - public void persist(C entity, DbGridMetadata identifier) throws SQLException { - if (entity instanceof InputEntity inputEntity) { - persistIncludeNested(inputEntity, identifier); - } else if (entity instanceof ResultEntity resultEntity) { - insert(resultEntity, identifier); - } else if (entity instanceof TimeSeries timeSeries) { - persistTimeSeries(timeSeries, identifier); - } else { - log.error("I don't know how to handle an entity of class {}", entity.getClass().getSimpleName()); - } - } - - /** - * Persist an entity. In contrast to {@link SqlSink#persist} this function does not extract nested entities. - * @param entity the entity that should be persisted - * @param identifier identifier of the grid - * @throws SQLException - */ - public void persistIgnoreNested(C entity, DbGridMetadata identifier) throws SQLException { - insert(entity, identifier); + } + }); + + // Persist the entities in hierarchic order to avoid failure because of foreign keys + for (Class cls : hierarchicInsert()) { + persistMixedList( + entitiesToAdd.stream() + .filter(ent -> cls.isAssignableFrom(ent.getClass())) + .collect(Collectors.toList()), + identifier); + entitiesToAdd.removeIf( + ent -> + cls.isAssignableFrom( + ent.getClass())); // maybe it's not necessary but I'm not sure if there are + // entities who aren't in the hierarchic structure } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - protected void persistListIncludeNested(List entities, Class cls, DbGridMetadata identifier) throws SQLException { - if (NestedEntity.class.isAssignableFrom(cls)) { - entities.forEach( - entity -> { - try { - List arr = new ArrayList<>(Extractor.extractElements((NestedEntity) entity)); - persistAll(arr, identifier); - } catch (ExtractorException | SQLException e) { - log.error( - String.format( - "An error occurred during extraction of nested entity'%s': ", - cls.getSimpleName()), - e); - } - } - ); - insertListIgnoreNested(entities, cls, identifier); - } else { - insertListIgnoreNested(entities, cls, identifier); - } + persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities + } + + /** + * Persist multiple input entities in a collection. In contrast to {@link SqlSink#persistAll} this + * function does not extract nested entities. + * + * @param entities a collection of entities that should be persisted + * @param identifier identifier of the grid + */ + public void persistAllIgnoreNested( + Collection entities, DbGridMetadata identifier) { + persistMixedList(new ArrayList<>(entities), identifier); + } + + /** + * Persist an entity. By default this method take care about the extraction process of nested + * entities (if any) + * + * @param entity the entity that should be persisted + * @param identifier identifier of the grid + * @throws SQLException + */ + public void persist(C entity, DbGridMetadata identifier) + throws SQLException { + if (entity instanceof InputEntity inputEntity) { + persistIncludeNested(inputEntity, identifier); + } else if (entity instanceof ResultEntity resultEntity) { + insert(resultEntity, identifier); + } else if (entity instanceof TimeSeries timeSeries) { + persistTimeSeries(timeSeries, identifier); + } else { + log.error( + "I don't know how to handle an entity of class {}", entity.getClass().getSimpleName()); } - - public void persistIncludeNested(C entity, DbGridMetadata identifier) throws SQLException { - Set entitiesToAdd = new HashSet<>(); - entitiesToAdd.add(entity); - persistAll(entitiesToAdd, identifier); - } - - private void persistMixedList(List entities, DbGridMetadata identifier) { - Map, List> entitiesPerClass = entities.stream() - .collect(groupingBy(entity -> (Class) entity.getClass())); - entitiesPerClass.forEach( - (cls, ent) -> { - try { - persistList(ent, cls, identifier); - } catch (SQLException e) { - throw new RuntimeException(String.format( - "An error occurred during extraction of entity '%s', SQLReason: '%s'", - cls.getSimpleName(), e.getMessage()), e); - } - } - ); - } - - private , V extends Value> void persistList(List entities, Class cls, DbGridMetadata identifier) throws SQLException { - // Check if there are only elements of the same class - Class firstClass = entities.get(0).getClass(); - boolean allSameClass = entities.stream().allMatch(e -> e.getClass() == firstClass); - - if (allSameClass) { - if (InputEntity.class.isAssignableFrom(cls)) { - insertListIgnoreNested(entities, cls, identifier); - } else if (ResultEntity.class.isAssignableFrom(cls)) { - insertListIgnoreNested(entities, cls, identifier); - } else if (TimeSeries.class.isAssignableFrom(cls)) { - entities.forEach(ts -> { - try { - persistTimeSeries((TimeSeries) ts, identifier); - } catch (SQLException e) { - throw new RuntimeException(String.format( - "An error occurred during extraction of entity '%s', SQLReason: '%s'", - cls.getSimpleName(), e.getMessage()), e); - } - }); - } else { - log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); - } - } else { - log.error("The list isn't homogenous regarding the classes of the elements."); - } - } - - /** - * Writes a list of entities into a sql table. It's necessary that all entities have the same class. - */ - private void insertListIgnoreNested(List entities, Class cls, DbGridMetadata identifier) throws SQLException { - try { - String[] headerElements = processorProvider.getHeaderElements(cls); - String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); - query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); - connector.executeUpdate(query); - } catch (ProcessorProviderException e) { - log.error("Exception occurred during processor request: ", e); - } - } - - protected , V extends Value> void persistTimeSeries(TimeSeries timeSeries, DbGridMetadata identifier) throws SQLException { - try { - TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); - String[] headerElements = processorProvider.getHeaderElements(key); - persistTimeSeries(timeSeries, headerElements, identifier); - } catch (ProcessorProviderException e) { - log.error( - "Exception occurred during receiving of header elements. Cannot write this element.", e); - } catch (IOException e) { - log.error("Exception occurred during closing of writer.", e); - } - } - - private , V extends Value> void persistTimeSeries( - TimeSeries timeSeries, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException, IOException, SQLException { - try { - String query = basicInsertQueryValuesITS(schemaName, databaseNamingStrategy.getEntityName(timeSeries).orElseThrow(), headerElements); - Set> entityFieldData = - processorProvider.handleTimeSeries(timeSeries); - query = query + entityFieldData.stream().map( - data -> queryTimeSeriesValueLine(sqlEntityFieldData(data), headerElements, identifier, timeSeries.getUuid().toString())).collect(Collectors.joining(",\n", "", ";")); + } + + /** + * Persist an entity. In contrast to {@link SqlSink#persist} this function does not extract nested + * entities. + * + * @param entity the entity that should be persisted + * @param identifier identifier of the grid + * @throws SQLException + */ + public void persistIgnoreNested(C entity, DbGridMetadata identifier) + throws SQLException { + insert(entity, identifier); + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + protected void persistListIncludeNested( + List entities, Class cls, DbGridMetadata identifier) throws SQLException { + if (NestedEntity.class.isAssignableFrom(cls)) { + entities.forEach( + entity -> { try { - connector.executeUpdate(query); - } catch (SQLException e) { - throw new RuntimeException(String.format( - "An error occurred during extraction of the time series, SQLReason: '%s'", - e.getMessage()), e); + List arr = + new ArrayList<>(Extractor.extractElements((NestedEntity) entity)); + persistAll(arr, identifier); + } catch (ExtractorException | SQLException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + cls.getSimpleName()), + e); } - } catch (ProcessorProviderException e) { - throw new ProcessorProviderException("Exception occurred during processor request: ", e); - } - } - - public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) throws SQLException { - DbGridMetadata identifier = new DbGridMetadata( - jointGridContainer.getGridName(), gridUUID - ); - - // get raw grid entities with types or operators - RawGridElements rawGridElements = jointGridContainer.getRawGrid(); - Set nodes = rawGridElements.getNodes(); - Set lines = rawGridElements.getLines(); - Set transformer2Ws = rawGridElements.getTransformer2Ws(); - Set transformer3Ws = rawGridElements.getTransformer3Ws(); - Set switches = rawGridElements.getSwitches(); - Set measurementUnits = rawGridElements.getMeasurementUnits(); - - // get system participants with types or operators - SystemParticipants systemParticipants = jointGridContainer.getSystemParticipants(); - Set bmPlants = systemParticipants.getBmPlants(); - Set chpPlants = systemParticipants.getChpPlants(); - Set evCS = systemParticipants.getEvCS(); - Set evs = systemParticipants.getEvs(); - Set fixedFeedIns = systemParticipants.getFixedFeedIns(); - Set heatPumps = systemParticipants.getHeatPumps(); - Set loads = systemParticipants.getLoads(); - Set pvPlants = systemParticipants.getPvPlants(); - Set storages = systemParticipants.getStorages(); - Set wecPlants = systemParticipants.getWecPlants(); - Set emSystems = systemParticipants.getEmSystems(); - - // get graphic elements (just for better readability, we could also just get them directly - // below) - GraphicElements graphicElements = jointGridContainer.getGraphics(); - - // extract types - Set types = - Stream.of( - lines, - transformer2Ws, - transformer3Ws, - bmPlants, - chpPlants, - evs, - heatPumps, - storages, - wecPlants) - .flatMap(Collection::stream) - .map(Extractor::extractType) - .collect(Collectors.toSet()); - - // extract operators - Set operators = - Stream.of( - nodes, - lines, - transformer2Ws, - transformer3Ws, - switches, - measurementUnits, - bmPlants, - chpPlants, - evCS, - evs, - fixedFeedIns, - heatPumps, - loads, - pvPlants, - storages, - wecPlants, - emSystems) - .flatMap(Collection::stream) - .map(Extractor::extractOperator) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - - List toAdd = new LinkedList<>(); - toAdd.addAll(rawGridElements.allEntitiesAsList()); - toAdd.addAll(systemParticipants.allEntitiesAsList()); - toAdd.addAll(graphicElements.allEntitiesAsList()); - toAdd.addAll(types); - toAdd.addAll(operators); - - // persist all entities - persistAll(toAdd, identifier); - } - private void insert(C entity, DbGridMetadata identifier) throws SQLException { - try { - String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); - String query = basicInsertQueryValuesGrid(schemaName, databaseNamingStrategy.getEntityName(entity.getClass()).orElseThrow(), headerElements) + queryValueLine(entity, headerElements, identifier) + ";"; - connector.executeUpdate(query); - } catch (ProcessorProviderException e) { - log.error("Exception occurred during receiving of header elements. Cannot write this element.", e); - } + }); + insertListIgnoreNested(entities, cls, identifier); + } else { + insertListIgnoreNested(entities, cls, identifier); } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - /** - * Provides the value lists for an insertion query. Conflicts because of the primary key 'uuid' will be ignored. Conflicts can occur if an entity (e.g. node) already exist. - * WARNING: It's assumed that all entities are from the same class C. - */ - private String createInsertQueryBodyIgnoreConflict(List entities, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { - Set> entityFieldData = - processorProvider - .handleEntities(entities); - String queryBody = ""; - queryBody = queryBody + entityFieldData.stream().map( - data -> - queryValueLine(sqlEntityFieldData(data), headerElements, identifier) - ).collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); - return queryBody; + } + + public void persistIncludeNested(C entity, DbGridMetadata identifier) + throws SQLException { + Set entitiesToAdd = new HashSet<>(); + entitiesToAdd.add(entity); + persistAll(entitiesToAdd, identifier); + } + + private void persistMixedList( + List entities, DbGridMetadata identifier) { + Map, List> entitiesPerClass = + entities.stream().collect(groupingBy(entity -> (Class) entity.getClass())); + entitiesPerClass.forEach( + (cls, ent) -> { + try { + persistList(ent, cls, identifier); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "An error occurred during extraction of entity '%s', SQLReason: '%s'", + cls.getSimpleName(), e.getMessage()), + e); + } + }); + } + + private , V extends Value> void persistList( + List entities, Class cls, DbGridMetadata identifier) throws SQLException { + // Check if there are only elements of the same class + Class firstClass = entities.get(0).getClass(); + boolean allSameClass = entities.stream().allMatch(e -> e.getClass() == firstClass); + + if (allSameClass) { + if (InputEntity.class.isAssignableFrom(cls)) { + insertListIgnoreNested(entities, cls, identifier); + } else if (ResultEntity.class.isAssignableFrom(cls)) { + insertListIgnoreNested(entities, cls, identifier); + } else if (TimeSeries.class.isAssignableFrom(cls)) { + entities.forEach( + ts -> { + try { + persistTimeSeries((TimeSeries) ts, identifier); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "An error occurred during extraction of entity '%s', SQLReason: '%s'", + cls.getSimpleName(), e.getMessage()), + e); + } + }); + } else { + log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); + } + } else { + log.error("The list isn't homogenous regarding the classes of the elements."); } - - /** - * Provides the value lists for an insertion query. - * - * WARNING: It's assumed that all entities are from the same class C. - */ - private String createInsertQueryBody(List entities, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { - Set> entityFieldData = - processorProvider - .handleEntities(entities); - String queryBody = ""; - queryBody = queryBody + entityFieldData.stream().map( - data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)).collect(Collectors.joining(",\n", "", ";")); - return queryBody; + } + + /** + * Writes a list of entities into a sql table. It's necessary that all entities have the same + * class. + */ + private void insertListIgnoreNested( + List entities, Class cls, DbGridMetadata identifier) throws SQLException { + try { + String[] headerElements = processorProvider.getHeaderElements(cls); + String query = + basicInsertQueryValuesGrid( + schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); + query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); + connector.executeUpdate(query); + } catch (ProcessorProviderException e) { + log.error("Exception occurred during processor request: ", e); } - - /** - * Creates a line with the values of one entity for an insertion query using the entityFieldData. - */ - private String queryValueLine(LinkedHashMap entityFieldData, String[] headerElements, DbGridMetadata identifier) { - return writeOneLine(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery())); - } - - /** - * Creates a line with the values of one entity for an insertion query. - */ - private String queryValueLine(C entity, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { - return queryValueLine(processorProvider.handleEntity(entity).map(this::sqlEntityFieldData).getOrThrow(), headerElements, identifier); + } + + protected , V extends Value> void persistTimeSeries( + TimeSeries timeSeries, DbGridMetadata identifier) throws SQLException { + try { + TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); + String[] headerElements = processorProvider.getHeaderElements(key); + persistTimeSeries(timeSeries, headerElements, identifier); + } catch (ProcessorProviderException e) { + log.error( + "Exception occurred during receiving of header elements. Cannot write this element.", e); + } catch (IOException e) { + log.error("Exception occurred during closing of writer.", e); } - - private String queryTimeSeriesValueLine(Map entityFieldData, String[] headerElements, DbGridMetadata identifier, String TSuuid) { - return writeOneLine(Stream.concat(Stream.concat(Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery()), Stream.of(quote(TSuuid,"'")))); - } - - private LinkedHashMap sqlEntityFieldData( - LinkedHashMap entityFieldData - ) { - LinkedHashMap quotedEntityFieldData = new LinkedHashMap<>(entityFieldData); - quotedEntityFieldData.replaceAll( - (key, ent) -> quote(ent, "'") - ); - - return quotedEntityFieldData; - } - - /** - * "INSERT INTO" line with schemaName.tableName - */ - private static String basicInsertQuery(String schemaName, String tableName) { - return "INSERT INTO\n\t" + schemaName + "." + tableName; - } - - /** - * Provides the insert, column names, grid identifier and the VALUES statement for a query. - */ - private String basicInsertQueryValuesGrid(String schemaName, String tableName, String[] headerElements) { - String[] addParams = {DbGridMetadata.GRID_UUID}; - return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; - } - - /** - * Provides the insert, column names, grid identifier, time_series uuid and the VALUES statement for a query. - */ - private String basicInsertQueryValuesITS(String schemaName, String tableName, String[] headerElements) { - String[] addParams = { - DbGridMetadata.GRID_UUID, - TIME_SERIES}; - return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + "\nVALUES\n"; - } - - /** - * Converts a stream of strings into an one line string with brackets. - */ - private String writeOneLine(Stream entries) { - return "(" + entries.collect(Collectors.joining(",")) + ")"; - } - - /** - * Converts an array of strings and an array of strings (for additional parameters) into an one line string with brackets. - */ - private String writeOneLine(String[] entries, String[] addParams) { - return writeOneLine(Stream.concat(Arrays.stream(entries), Arrays.stream(addParams))); - } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - public void createClassTable( - Class cls - ) throws SQLException, ProcessorProviderException, EntityProcessorException { - String query = getDataTypes(cls); + } + + private , V extends Value> void persistTimeSeries( + TimeSeries timeSeries, String[] headerElements, DbGridMetadata identifier) + throws ProcessorProviderException, IOException, SQLException { + try { + String query = + basicInsertQueryValuesITS( + schemaName, + databaseNamingStrategy.getEntityName(timeSeries).orElseThrow(), + headerElements); + Set> entityFieldData = + processorProvider.handleTimeSeries(timeSeries); + query = + query + + entityFieldData.stream() + .map( + data -> + queryTimeSeriesValueLine( + sqlEntityFieldData(data), + headerElements, + identifier, + timeSeries.getUuid().toString())) + .collect(Collectors.joining(",\n", "", ";")); + try { connector.executeUpdate(query); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "An error occurred during extraction of the time series, SQLReason: '%s'", + e.getMessage()), + e); + } + } catch (ProcessorProviderException e) { + throw new ProcessorProviderException("Exception occurred during processor request: ", e); } - - public void createGridTable( - String schemaName, String tableName - ) throws SQLException { - String query = queryForGridTable(schemaName, tableName); - connector.executeUpdate(query); - } - - /** - * @return insertion order for unique entities - */ - private static List> hierarchicInsert() { - List> sortedInsert = new ArrayList<>(); - sortedInsert.add(AssetTypeInput.class); //1. Types - sortedInsert.add(OperatorInput.class); //2. Operators - sortedInsert.add(NodeInput.class); //3. Nodes - sortedInsert.add(ThermalBusInput.class); //4. ThermalBus - sortedInsert.add(ThermalUnitInput.class); //5. ThermalUnit - sortedInsert.add(ConnectorInput.class); //6a. ConnectorInput - sortedInsert.add(SystemParticipantInput.class); //6b. SystemParticipantInput - sortedInsert.add(GraphicInput.class); //7. GraphicInput - //8. Rest - return sortedInsert; + } + + public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) + throws SQLException { + DbGridMetadata identifier = new DbGridMetadata(jointGridContainer.getGridName(), gridUUID); + + // get raw grid entities with types or operators + RawGridElements rawGridElements = jointGridContainer.getRawGrid(); + Set nodes = rawGridElements.getNodes(); + Set lines = rawGridElements.getLines(); + Set transformer2Ws = rawGridElements.getTransformer2Ws(); + Set transformer3Ws = rawGridElements.getTransformer3Ws(); + Set switches = rawGridElements.getSwitches(); + Set measurementUnits = rawGridElements.getMeasurementUnits(); + + // get system participants with types or operators + SystemParticipants systemParticipants = jointGridContainer.getSystemParticipants(); + Set bmPlants = systemParticipants.getBmPlants(); + Set chpPlants = systemParticipants.getChpPlants(); + Set evCS = systemParticipants.getEvCS(); + Set evs = systemParticipants.getEvs(); + Set fixedFeedIns = systemParticipants.getFixedFeedIns(); + Set heatPumps = systemParticipants.getHeatPumps(); + Set loads = systemParticipants.getLoads(); + Set pvPlants = systemParticipants.getPvPlants(); + Set storages = systemParticipants.getStorages(); + Set wecPlants = systemParticipants.getWecPlants(); + Set emSystems = systemParticipants.getEmSystems(); + + // get graphic elements (just for better readability, we could also just get them directly + // below) + GraphicElements graphicElements = jointGridContainer.getGraphics(); + + // extract types + Set types = + Stream.of( + lines, + transformer2Ws, + transformer3Ws, + bmPlants, + chpPlants, + evs, + heatPumps, + storages, + wecPlants) + .flatMap(Collection::stream) + .map(Extractor::extractType) + .collect(Collectors.toSet()); + + // extract operators + Set operators = + Stream.of( + nodes, + lines, + transformer2Ws, + transformer3Ws, + switches, + measurementUnits, + bmPlants, + chpPlants, + evCS, + evs, + fixedFeedIns, + heatPumps, + loads, + pvPlants, + storages, + wecPlants, + emSystems) + .flatMap(Collection::stream) + .map(Extractor::extractOperator) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + + List toAdd = new LinkedList<>(); + toAdd.addAll(rawGridElements.allEntitiesAsList()); + toAdd.addAll(systemParticipants.allEntitiesAsList()); + toAdd.addAll(graphicElements.allEntitiesAsList()); + toAdd.addAll(types); + toAdd.addAll(operators); + + // persist all entities + persistAll(toAdd, identifier); + } + + private void insert(C entity, DbGridMetadata identifier) + throws SQLException { + try { + String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); + String query = + basicInsertQueryValuesGrid( + schemaName, + databaseNamingStrategy.getEntityName(entity.getClass()).orElseThrow(), + headerElements) + + queryValueLine(entity, headerElements, identifier) + + ";"; + connector.executeUpdate(query); + } catch (ProcessorProviderException e) { + log.error( + "Exception occurred during receiving of header elements. Cannot write this element.", e); } + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + /** + * Provides the value lists for an insertion query. Conflicts because of the primary key 'uuid' + * will be ignored. Conflicts can occur if an entity (e.g. node) already exist. WARNING: It's + * assumed that all entities are from the same class C. + */ + private String createInsertQueryBodyIgnoreConflict( + List entities, String[] headerElements, DbGridMetadata identifier) + throws ProcessorProviderException { + Set> entityFieldData = processorProvider.handleEntities(entities); + String queryBody = ""; + queryBody = + queryBody + + entityFieldData.stream() + .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) + .collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + return queryBody; + } + + /** + * Provides the value lists for an insertion query. + * + *

WARNING: It's assumed that all entities are from the same class C. + */ + private String createInsertQueryBody( + List entities, String[] headerElements, DbGridMetadata identifier) + throws ProcessorProviderException { + Set> entityFieldData = processorProvider.handleEntities(entities); + String queryBody = ""; + queryBody = + queryBody + + entityFieldData.stream() + .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) + .collect(Collectors.joining(",\n", "", ";")); + return queryBody; + } + + /** + * Creates a line with the values of one entity for an insertion query using the entityFieldData. + */ + private String queryValueLine( + LinkedHashMap entityFieldData, + String[] headerElements, + DbGridMetadata identifier) { + return writeOneLine( + Stream.concat( + Arrays.stream(headerElements).map(entityFieldData::get), + identifier.getStreamForQuery())); + } + + /** Creates a line with the values of one entity for an insertion query. */ + private String queryValueLine( + C entity, String[] headerElements, DbGridMetadata identifier) + throws ProcessorProviderException { + return queryValueLine( + processorProvider.handleEntity(entity).map(this::sqlEntityFieldData).getOrThrow(), + headerElements, + identifier); + } + + private String queryTimeSeriesValueLine( + Map entityFieldData, + String[] headerElements, + DbGridMetadata identifier, + String TSuuid) { + return writeOneLine( + Stream.concat( + Stream.concat( + Arrays.stream(headerElements).map(entityFieldData::get), + identifier.getStreamForQuery()), + Stream.of(quote(TSuuid, "'")))); + } + + private LinkedHashMap sqlEntityFieldData( + LinkedHashMap entityFieldData) { + LinkedHashMap quotedEntityFieldData = new LinkedHashMap<>(entityFieldData); + quotedEntityFieldData.replaceAll((key, ent) -> quote(ent, "'")); + + return quotedEntityFieldData; + } + + /** "INSERT INTO" line with schemaName.tableName */ + private static String basicInsertQuery(String schemaName, String tableName) { + return "INSERT INTO\n\t" + schemaName + "." + tableName; + } + + /** Provides the insert, column names, grid identifier and the VALUES statement for a query. */ + private String basicInsertQueryValuesGrid( + String schemaName, String tableName, String[] headerElements) { + String[] addParams = {DbGridMetadata.GRID_UUID}; + return basicInsertQuery(schemaName, tableName) + + " " + + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + + "\nVALUES\n"; + } + + /** + * Provides the insert, column names, grid identifier, time_series uuid and the VALUES statement + * for a query. + */ + private String basicInsertQueryValuesITS( + String schemaName, String tableName, String[] headerElements) { + String[] addParams = {DbGridMetadata.GRID_UUID, TIME_SERIES}; + return basicInsertQuery(schemaName, tableName) + + " " + + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) + + "\nVALUES\n"; + } + + /** Converts a stream of strings into an one line string with brackets. */ + private String writeOneLine(Stream entries) { + return "(" + entries.collect(Collectors.joining(",")) + ")"; + } + + /** + * Converts an array of strings and an array of strings (for additional parameters) into an one + * line string with brackets. + */ + private String writeOneLine(String[] entries, String[] addParams) { + return writeOneLine(Stream.concat(Arrays.stream(entries), Arrays.stream(addParams))); + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + + public void createClassTable(Class cls) + throws SQLException, ProcessorProviderException, EntityProcessorException { + String query = getDataTypes(cls); + connector.executeUpdate(query); + } + + public void createGridTable(String schemaName, String tableName) throws SQLException { + String query = queryForGridTable(schemaName, tableName); + connector.executeUpdate(query); + } + + /** @return insertion order for unique entities */ + private static List> hierarchicInsert() { + List> sortedInsert = new ArrayList<>(); + sortedInsert.add(AssetTypeInput.class); // 1. Types + sortedInsert.add(OperatorInput.class); // 2. Operators + sortedInsert.add(NodeInput.class); // 3. Nodes + sortedInsert.add(ThermalBusInput.class); // 4. ThermalBus + sortedInsert.add(ThermalUnitInput.class); // 5. ThermalUnit + sortedInsert.add(ConnectorInput.class); // 6a. ConnectorInput + sortedInsert.add(SystemParticipantInput.class); // 6b. SystemParticipantInput + sortedInsert.add(GraphicInput.class); // 7. GraphicInput + // 8. Rest + return sortedInsert; + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index cba22bd12..bbff8af0d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -1,3 +1,8 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ package edu.ie3.datamodel.io.sink import edu.ie3.datamodel.io.DbGridMetadata @@ -67,307 +72,306 @@ import static tech.units.indriya.unit.Units.PERCENT @Testcontainers class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeriesTestData { - @Shared - PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:14.2") + @Shared + PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:14.2") - @Shared - SqlConnector connector + @Shared + SqlConnector connector - @Shared - SqlDataSource sqlSource + @Shared + SqlDataSource sqlSource - @Shared - DatabaseNamingStrategy namingStrategy + @Shared + DatabaseNamingStrategy namingStrategy - @Shared - DbGridMetadata identifier + @Shared + DbGridMetadata identifier - static String schemaName = "public" + static String schemaName = "public" - def setupSpec() { - // Copy sql import scripts into docker - MountableFile sqlImportFile = getMountableFile("_sql/") - postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/") - postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "setup.sql") + def setupSpec() { + // Copy sql import scripts into docker + MountableFile sqlImportFile = getMountableFile("_sql/") + postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/") + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "setup.sql") - connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) + connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - namingStrategy = new DatabaseNamingStrategy() + namingStrategy = new DatabaseNamingStrategy() - identifier = new DbGridMetadata("vn_simona", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120002")) - - sqlSource = new SqlDataSource(connector, schemaName, namingStrategy) - } - - def setup() { - // Execute import script - Iterable importFiles = Arrays.asList( - "grids.sql", - "types.sql", - "result_entities.sql", - "input_entities.sql", - "time_series.sql", - "load_profile.sql" - ) - for (String file: importFiles) { - Container.ExecResult res = postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + file) - assert res.stderr.empty - } - } - - def cleanup() { - postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "cleanup.sql") - } - - def "SQL sink can persist provided elements correctly"() { - given: - TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) - TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) - HashMap timeSeriesProcessorMap = new HashMap<>() - timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) - IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries - - SqlSink sink = new SqlSink(schemaName, - new ProcessorProvider([ - new ResultEntityProcessor(PvResult), - new ResultEntityProcessor(WecResult), - new ResultEntityProcessor(EvResult), - new ResultEntityProcessor(EvcsResult), - new ResultEntityProcessor(EmResult), - new ResultEntityProcessor(FlexOptionsResult), - new InputEntityProcessor(Transformer2WInput), - new InputEntityProcessor(NodeInput), - new InputEntityProcessor(EvcsInput), - new InputEntityProcessor(Transformer2WTypeInput), - new InputEntityProcessor(LineGraphicInput), - new InputEntityProcessor(NodeGraphicInput), - new InputEntityProcessor(CylindricalStorageInput), - new InputEntityProcessor(ThermalHouseInput), - new InputEntityProcessor(OperatorInput), - new InputEntityProcessor(LineInput), - new InputEntityProcessor(ThermalBusInput), - new InputEntityProcessor(LineTypeInput), - new InputEntityProcessor(LoadInput), - new InputEntityProcessor(EmInput) - ], timeSeriesProcessorMap), - namingStrategy, - connector) - UUID uuid = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") - UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") - Quantity p = Quantities.getQuantity(10, StandardUnits.ACTIVE_POWER_IN) - Quantity q = Quantities.getQuantity(10, StandardUnits.REACTIVE_POWER_IN) - PvResult pvResult = new PvResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - WecResult wecResult = new WecResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - EvcsResult evcsResult = new EvcsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - EmResult emResult = new EmResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - - Quantity pRef = Quantities.getQuantity(5.1, StandardUnits.ACTIVE_POWER_RESULT) - Quantity pMin = Quantities.getQuantity(-6, StandardUnits.ACTIVE_POWER_RESULT) - Quantity pMax = Quantities.getQuantity(6, StandardUnits.ACTIVE_POWER_RESULT) - FlexOptionsResult flexOptionsResult = new FlexOptionsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, pRef, pMin, pMax) - - when: - sink.persistAll([ - pvResult, - wecResult, - evcsResult, - emResult, - flexOptionsResult, - GridTestData.transformerCtoG, - GridTestData.lineGraphicCtoD, - GridTestData.nodeGraphicC, - ThermalUnitInputTestData.cylindricStorageInput, - ThermalUnitInputTestData.thermalHouseInput, - SystemParticipantTestData.evcsInput, - SystemParticipantTestData.loadInput, - SystemParticipantTestData.emInput, - individualTimeSeries - ], identifier) - - then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_res", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_res", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "evcs_res", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_res", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "flex_options_res", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 4 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 2 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "cylindrical_storage_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_graphic_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_graphic_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_bus_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_house_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "ev_res", ps -> {}).count() == 0 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 - - cleanup: - sink.shutdown() - } - - - def "A SqlSink can persist a time series."() { - given: - TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) - TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) - HashMap timeSeriesProcessorMap = new HashMap<>() - timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) - IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries - SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) - - when: - sink.persist(individualTimeSeries, identifier) - - then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 - - cleanup: - sink.shutdown() - } - - def "A valid SqlSink persists a bunch of time series correctly"() { - given: - SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) - SqlDataSource source = new SqlDataSource(connector, schemaName, namingStrategy) - - when: - sink.persistAll(allTimeSeries, identifier) - - then: - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_p", ps -> {}).count() == 3 - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pq", ps -> {}).count() == 3 - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_ph", ps -> {}).count() == 3 - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pqh", ps -> {}).count() == 3 - source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_weather", ps -> {}).count() == 3 - - cleanup: - sink.shutdown() - } - - def "A valid SqlSink throws an exception if an entity has null for a not null attribute."() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - def nestedInput = new PvInput( - UUID.fromString("d56f15b7-8293-4b98-b5bd-58f6273ce229"), - "test_pvInput", - OperatorInput.NO_OPERATOR_ASSIGNED, - OperationTime.notLimited(), - Mock(NodeInput), - new CosPhiFixed("cosPhiFixed:{(0.0,0.95)}"), - 0.2, - Quantities.getQuantity(-8.926613807678223, DEGREE_GEOM), - Quantities.getQuantity(95d, PERCENT), - Quantities.getQuantity(41.01871871948242, DEGREE_GEOM), - 0.8999999761581421, - 1, - false, - Quantities.getQuantity(25d, KILOVOLTAMPERE), - 0.95 - ) - - when: - sink.persistIgnoreNested(nestedInput, identifier) - - then: - def exception = thrown(SQLException) - exception.message.contains("ERROR: invalid input syntax for type uuid: \"null\"\n") - - cleanup: - sink.shutdown() - } + identifier = new DbGridMetadata("vn_simona", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120002")) + sqlSource = new SqlDataSource(connector, schemaName, namingStrategy) + } - def "A valid SqlSink refuses to persist an entity, if no processor can be found for a specific input"() { - given: - def sink = new SqlSink( - schemaName, - new ProcessorProvider( - ProcessorProvider.allEntityProcessors(), - new HashMap, Value>, TimeSeriesEntry, Value>>()), - namingStrategy, - connector) - - when: - sink.persist(individualEnergyPriceTimeSeries, identifier) - - then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 0 - - cleanup: - sink.shutdown() + def setup() { + // Execute import script + Iterable importFiles = Arrays.asList( + "grids.sql", + "types.sql", + "result_entities.sql", + "input_entities.sql", + "time_series.sql", + "load_profile.sql" + ) + for (String file: importFiles) { + Container.ExecResult res = postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + file) + assert res.stderr.empty } - - def "A valid SqlSink throws an exception if a nested entity hasn't all of its nested entity."() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - - when: - sink.persistIgnoreNested(SystemParticipantTestData.loadInput, identifier) - - then: - def exception = thrown(SQLException) - exception.message == "ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + - " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." - - cleanup: - sink.shutdown() - } - - def "A valid SqlSink can create a table for class."() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - def wec = SystemParticipantTestData.wecInput - - when: - sink.createClassTable(wec.getClass()) - - then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_input", ps -> {}).count() == 0 - } - - def "A valid SqlSink throws an exception if a grid does not exist."() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - - when: - def failIdentifier = new DbGridMetadata("fail_grid", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120003")) - - sink.persist(individualEnergyPriceTimeSeries, failIdentifier) - - then: - def exception = thrown(RuntimeException) - exception.message.contains("Detail: Key (grid_uuid)=(8e6bd444-4580-11ee-be56-0242ac120003) is not present in table \"grids\".") - - cleanup: - sink.shutdown() - } - - def "A valid SqlSink should persist a valid joint grid container correctly"() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - - when: - sink.persistJointGrid(SampleJointGrid.grid(), UUID.fromString("297dfac8-83cc-11ee-b962-0242ac120002")) - - then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 6 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_type_input", ps -> {}).count() == 2 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 2 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 7 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_type_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 2 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 2 - - cleanup: - sink.shutdown() - } - + } + + def cleanup() { + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/" + "cleanup.sql") + } + + def "SQL sink can persist provided elements correctly"() { + given: + TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + HashMap timeSeriesProcessorMap = new HashMap<>() + timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) + IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries + + SqlSink sink = new SqlSink(schemaName, + new ProcessorProvider([ + new ResultEntityProcessor(PvResult), + new ResultEntityProcessor(WecResult), + new ResultEntityProcessor(EvResult), + new ResultEntityProcessor(EvcsResult), + new ResultEntityProcessor(EmResult), + new ResultEntityProcessor(FlexOptionsResult), + new InputEntityProcessor(Transformer2WInput), + new InputEntityProcessor(NodeInput), + new InputEntityProcessor(EvcsInput), + new InputEntityProcessor(Transformer2WTypeInput), + new InputEntityProcessor(LineGraphicInput), + new InputEntityProcessor(NodeGraphicInput), + new InputEntityProcessor(CylindricalStorageInput), + new InputEntityProcessor(ThermalHouseInput), + new InputEntityProcessor(OperatorInput), + new InputEntityProcessor(LineInput), + new InputEntityProcessor(ThermalBusInput), + new InputEntityProcessor(LineTypeInput), + new InputEntityProcessor(LoadInput), + new InputEntityProcessor(EmInput) + ], timeSeriesProcessorMap), + namingStrategy, + connector) + UUID uuid = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") + UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") + Quantity p = Quantities.getQuantity(10, StandardUnits.ACTIVE_POWER_IN) + Quantity q = Quantities.getQuantity(10, StandardUnits.REACTIVE_POWER_IN) + PvResult pvResult = new PvResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + WecResult wecResult = new WecResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + EvcsResult evcsResult = new EvcsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + EmResult emResult = new EmResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + + Quantity pRef = Quantities.getQuantity(5.1, StandardUnits.ACTIVE_POWER_RESULT) + Quantity pMin = Quantities.getQuantity(-6, StandardUnits.ACTIVE_POWER_RESULT) + Quantity pMax = Quantities.getQuantity(6, StandardUnits.ACTIVE_POWER_RESULT) + FlexOptionsResult flexOptionsResult = new FlexOptionsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, pRef, pMin, pMax) + + when: + sink.persistAll([ + pvResult, + wecResult, + evcsResult, + emResult, + flexOptionsResult, + GridTestData.transformerCtoG, + GridTestData.lineGraphicCtoD, + GridTestData.nodeGraphicC, + ThermalUnitInputTestData.cylindricStorageInput, + ThermalUnitInputTestData.thermalHouseInput, + SystemParticipantTestData.evcsInput, + SystemParticipantTestData.loadInput, + SystemParticipantTestData.emInput, + individualTimeSeries + ], identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "evcs_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "flex_options_res", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 4 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "cylindrical_storage_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_graphic_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_graphic_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_bus_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_house_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "ev_res", ps -> {}).count() == 0 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + + def "A SqlSink can persist a time series."() { + given: + TimeSeriesProcessor timeSeriesProcessor = new TimeSeriesProcessor<>(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) + HashMap timeSeriesProcessorMap = new HashMap<>() + timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) + IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries + SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persist(individualTimeSeries, identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink persists a bunch of time series correctly"() { + given: + SqlSink sink = new SqlSink(schemaName, namingStrategy, connector) + SqlDataSource source = new SqlDataSource(connector, schemaName, namingStrategy) + + when: + sink.persistAll(allTimeSeries, identifier) + + then: + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_p", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pq", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_ph", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_pqh", ps -> {}).count() == 3 + source.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_weather", ps -> {}).count() == 3 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink throws an exception if an entity has null for a not null attribute."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + def nestedInput = new PvInput( + UUID.fromString("d56f15b7-8293-4b98-b5bd-58f6273ce229"), + "test_pvInput", + OperatorInput.NO_OPERATOR_ASSIGNED, + OperationTime.notLimited(), + Mock(NodeInput), + new CosPhiFixed("cosPhiFixed:{(0.0,0.95)}"), + 0.2, + Quantities.getQuantity(-8.926613807678223, DEGREE_GEOM), + Quantities.getQuantity(95d, PERCENT), + Quantities.getQuantity(41.01871871948242, DEGREE_GEOM), + 0.8999999761581421, + 1, + false, + Quantities.getQuantity(25d, KILOVOLTAMPERE), + 0.95 + ) + + when: + sink.persistIgnoreNested(nestedInput, identifier) + + then: + def exception = thrown(SQLException) + exception.message.contains("ERROR: invalid input syntax for type uuid: \"null\"\n") + + cleanup: + sink.shutdown() + } + + + def "A valid SqlSink refuses to persist an entity, if no processor can be found for a specific input"() { + given: + def sink = new SqlSink( + schemaName, + new ProcessorProvider( + ProcessorProvider.allEntityProcessors(), + new HashMap, Value>, TimeSeriesEntry, Value>>()), + namingStrategy, + connector) + + when: + sink.persist(individualEnergyPriceTimeSeries, identifier) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 0 + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink throws an exception if a nested entity hasn't all of its nested entity."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persistIgnoreNested(SystemParticipantTestData.loadInput, identifier) + + then: + def exception = thrown(SQLException) + exception.message == "ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + + " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink can create a table for class."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + def wec = SystemParticipantTestData.wecInput + + when: + sink.createClassTable(wec.getClass()) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_input", ps -> {}).count() == 0 + } + + def "A valid SqlSink throws an exception if a grid does not exist."() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + def failIdentifier = new DbGridMetadata("fail_grid", UUID.fromString("8e6bd444-4580-11ee-be56-0242ac120003")) + + sink.persist(individualEnergyPriceTimeSeries, failIdentifier) + + then: + def exception = thrown(RuntimeException) + exception.message.contains("Detail: Key (grid_uuid)=(8e6bd444-4580-11ee-be56-0242ac120003) is not present in table \"grids\".") + + cleanup: + sink.shutdown() + } + + def "A valid SqlSink should persist a valid joint grid container correctly"() { + given: + def sink = new SqlSink(schemaName, namingStrategy, connector) + + when: + sink.persistJointGrid(SampleJointGrid.grid(), UUID.fromString("297dfac8-83cc-11ee-b962-0242ac120002")) + + then: + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_input", ps -> {}).count() == 6 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "line_type_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "node_input", ps -> {}).count() == 7 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "operator_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "pv_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "storage_type_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_type_input", ps -> {}).count() == 2 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "transformer_2_w_input", ps -> {}).count() == 2 + + cleanup: + sink.shutdown() + } } diff --git a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy index 0da55b6cd..2e9dd837f 100644 --- a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy @@ -55,8 +55,8 @@ trait TimeSeriesTestData { ) IndividualTimeSeriesMetaInformation individualEnergyPriceTimeSeriesMeta = new IndividualTimeSeriesMetaInformation( - UUID.fromString("a4bbcb77-b9d0-4b88-92be-b9a14a3e332b"), - ColumnScheme.ENERGY_PRICE + UUID.fromString("a4bbcb77-b9d0-4b88-92be-b9a14a3e332b"), + ColumnScheme.ENERGY_PRICE ) Set> individualEnergyPriceTimeSeriesProcessed = [ From b712f172711d0ae8b216e28df83491719649e1ef Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Wed, 22 Nov 2023 09:29:33 +0100 Subject: [PATCH 06/19] Improve SqlUtils --- .../edu/ie3/datamodel/io/DbGridMetadata.java | 1 + .../java/edu/ie3/datamodel/io/SqlUtils.java | 41 +++++++++++-------- .../edu/ie3/datamodel/io/sink/SqlSink.java | 8 ++-- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 2 +- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java index c9bf2fc2f..29df2bd90 100644 --- a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java +++ b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java @@ -13,6 +13,7 @@ /** Class for identification of entities and results from grids in SQL databases. */ public record DbGridMetadata(String gridName, UUID uuid) { + public static final String GRID_TABLE = "grids"; public static final String GRID_NAME = "grid_name"; public static final String GRID_UUID = "grid_uuid"; diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 358d3c83c..e78427934 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; public class SqlUtils { + protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; @@ -29,38 +30,35 @@ private SqlUtils() { throw new IllegalStateException("Utility classes cannot be instantiated"); } - public static String queryForCreation( - String schemaName, String tableName, Stream> columnsWithDataTypes) { - return beginQueryCreateTable(schemaName, tableName) + " " + endQueryCreateTable; - } - - public static String getEndQueryCreateTable() { - return endQueryCreateTable; - } - private static String beginQueryCreateTable(String schemaName, String tableName) { return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; } - public static String queryForGridTable(String schemaName, String tableName) { - return beginQueryCreateTable(schemaName, tableName) + /** @return query to create a SQL table for a grid */ + public static String queryCreateGridTable(String schemaName) { + return beginQueryCreateTable(schemaName, DbGridMetadata.GRID_TABLE) + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" + endQueryCreateTable; } - public static String getDataTypes(Class cls) + /** @return query to create a SQL table for an unique entity */ + public static String queryCreateTableUniqueEntity( + Class cls, String schemaName) throws EntityProcessorException, ProcessorProviderException { ProcessorProvider processorProvider = new ProcessorProvider(); DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); String[] headerElements = processorProvider.getHeaderElements(cls); - Stream strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_uuid")); + Stream strHeader = + Stream.concat(Arrays.stream(headerElements), Stream.of(DbGridMetadata.GRID_UUID)); Stream dtHeader = strHeader.map( element -> camelCaseToSnakeCase(element) + " " - + classToDataType().get(camelCaseToSnakeCase(element))); - return "CREATE TABLE public." + + columnToSqlDataType().get(camelCaseToSnakeCase(element))); + return "CREATE TABLE " + + schemaName + + "." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) @@ -70,7 +68,12 @@ public static String getDataTypes(Class cls) + "TABLESPACE pg_default;\n"; } - public static Map classToDataType() { + /** + * Map to create a SQL table for an entity with the right data types. + * + * @return Map column name -> data type + */ + public static Map columnToSqlDataType() { HashMap map = new HashMap(); map.put("uuid", "uuid PRIMARY KEY"); @@ -94,7 +97,6 @@ public static Map classToDataType() { map.put("length", "double precision NOT NULL"); map.put("node_a", "uuid NOT NULL"); map.put("node_b", "uuid NOT NULL"); - map.put("type", "uuid NOT NULL"); // EVCS map.put("olm_characteristic", "TEXT NOT NULL"); map.put("parallel_devices", "int NOT NULL"); map.put("cos_phi_rated", "TEXT NOT NULL"); @@ -182,6 +184,11 @@ public static Map classToDataType() { return map; } + /** + * To avoid data type conflicts while insertion into a SQL table all columns should be quoted. + * + * @return input with quoteSymbol + */ public static String quote(String input, String quoteSymbol) { if (input == "") { return "NULL"; diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 59d95b8ce..dee078f9d 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -544,14 +544,14 @@ private String writeOneLine(String[] entries, String[] addParams) { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - public void createClassTable(Class cls) + public void createClassTable(Class cls, String schemaName) throws SQLException, ProcessorProviderException, EntityProcessorException { - String query = getDataTypes(cls); + String query = queryCreateTableUniqueEntity(cls, schemaName); connector.executeUpdate(query); } - public void createGridTable(String schemaName, String tableName) throws SQLException { - String query = queryForGridTable(schemaName, tableName); + public void createGridTable(String schemaName) throws SQLException { + String query = queryCreateGridTable(schemaName); connector.executeUpdate(query); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index bbff8af0d..5997f65d6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -323,7 +323,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri sink.shutdown() } - def "A valid SqlSink can create a table for class."() { + def "A valid SqlSink can create a table for entity class."() { given: def sink = new SqlSink(schemaName, namingStrategy, connector) def wec = SystemParticipantTestData.wecInput From 75c57f9d4b4a6ccedee57136a696e753983fd99a Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Wed, 22 Nov 2023 16:08:19 +0100 Subject: [PATCH 07/19] Temprorary --- src/main/java/edu/ie3/datamodel/io/processor/Processor.java | 3 ++- src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java index 4d2a48454..11118e68a 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java @@ -19,6 +19,7 @@ import edu.ie3.datamodel.models.voltagelevels.VoltageLevel; import edu.ie3.datamodel.utils.Try; import edu.ie3.datamodel.utils.Try.*; +import edu.ie3.util.TimeUtil; import edu.ie3.util.exceptions.QuantityException; import java.beans.Introspector; import java.lang.reflect.InvocationTargetException; @@ -412,7 +413,7 @@ protected String processOperationTime(OperationTime operationTime, String fieldN * @return string representation of the ZonedDateTime */ protected String processZonedDateTime(ZonedDateTime zonedDateTime) { - return zonedDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss Z")); + return TimeUtil.withDefaults.toString(zonedDateTime); } /** diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index dee078f9d..b8e0c7a46 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -262,6 +262,7 @@ private void insertListIgnoreNested( basicInsertQueryValuesGrid( schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); + System.out.println(query); connector.executeUpdate(query); } catch (ProcessorProviderException e) { log.error("Exception occurred during processor request: ", e); From 4955b93c978976161aa74ef7a062b898c481c1f0 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Fri, 24 Nov 2023 10:27:45 +0100 Subject: [PATCH 08/19] temp --- src/main/java/edu/ie3/datamodel/io/processor/Processor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java index 11118e68a..86bdcaf5c 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java @@ -400,7 +400,6 @@ protected String processOperationTime(OperationTime operationTime, String fieldN operationTime .getEndDate() .ifPresent(endDate -> resultStringBuilder.append(processZonedDateTime(endDate))); - return resultStringBuilder.toString(); } From d85e2346421115ed892501e0def2900397893d03 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Tue, 18 Jun 2024 10:29:18 +0200 Subject: [PATCH 09/19] fixing tests --- .../java/edu/ie3/datamodel/io/SqlUtils.java | 4 +- .../datamodel/io/connectors/SqlConnector.java | 13 +++- .../io/naming/DatabaseNamingStrategy.java | 1 - .../io/processor/ProcessorProvider.java | 2 +- .../edu/ie3/datamodel/io/sink/SqlSink.java | 76 +++++++++++-------- .../io/source/sql/SqlDataSource.java | 7 +- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 39 +++++----- .../datamodel/io/sink/_sql/input_entities.sql | 18 +++-- .../datamodel/io/sink/_sql/load_profile.sql | 1 - .../io/sink/_sql/result_entities.sql | 6 -- .../datamodel/io/sink/_sql/time_series.sql | 7 -- .../edu/ie3/datamodel/io/sink/_sql/types.sql | 6 +- 12 files changed, 94 insertions(+), 86 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index e78427934..1c40edea4 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -11,6 +11,7 @@ import edu.ie3.datamodel.exceptions.ProcessorProviderException; import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.processor.ProcessorProvider; +import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.UniqueEntity; import java.util.Arrays; import java.util.HashMap; @@ -43,7 +44,7 @@ public static String queryCreateGridTable(String schemaName) { /** @return query to create a SQL table for an unique entity */ public static String queryCreateTableUniqueEntity( - Class cls, String schemaName) + Class cls, String schemaName) throws EntityProcessorException, ProcessorProviderException { ProcessorProvider processorProvider = new ProcessorProvider(); DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); @@ -103,6 +104,7 @@ public static Map columnToSqlDataType() { map.put("dsm", "bool NOT NULL"); map.put("e_cons_annual", "double precision NOT NULL"); map.put("load_profile", "TEXT NOT NULL"); + map.put("controlling_em", "uuid NOT NULL"); map.put("auto_tap", "bool NOT NULL"); map.put("tap_pos", "int NOT NULL"); diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index 91cf38dd9..f43ff8f8b 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -156,10 +156,15 @@ public Map extractFieldMap(ResultSet rs) { return insensitiveFieldsToAttributes; } - public boolean tableExists(Connection connection, String tableName) throws SQLException { - DatabaseMetaData meta = connection.getMetaData(); - ResultSet resultSet = meta.getTables(null, null, tableName, new String[] {"TABLE"}); + public boolean tableExistsSQL(String tableName) throws SQLException { + PreparedStatement preparedStatement = connection.prepareStatement("SELECT count(*) " + + "FROM information_schema.tables " + + "WHERE table_name = ?" + + "LIMIT 1;"); + preparedStatement.setString(1, tableName); - return resultSet.next(); + ResultSet resultSet = preparedStatement.executeQuery(); + resultSet.next(); + return resultSet.getInt(1) != 0; } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java index 2accf68ef..5e85512f8 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java @@ -8,7 +8,6 @@ import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.models.UniqueEntity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; diff --git a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java index d94757eaf..7064107f7 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java @@ -78,7 +78,7 @@ Try, ProcessorProviderException> handleEntity(T en .transformF(ProcessorProviderException::new)); } - public Set> handleEntities( + public Set> handleEntities( List entities) throws ProcessorProviderException { Set setOfEntities = new HashSet<>(entities); Set> setOfMaps = new HashSet<>(); diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index b8e0c7a46..59fe40c1f 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -17,7 +17,8 @@ import edu.ie3.datamodel.io.processor.EntityProcessor; import edu.ie3.datamodel.io.processor.ProcessorProvider; import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey; -import edu.ie3.datamodel.models.UniqueEntity; +import edu.ie3.datamodel.models.Entity; +import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.input.*; import edu.ie3.datamodel.models.input.connector.*; import edu.ie3.datamodel.models.input.container.GraphicElements; @@ -84,7 +85,7 @@ public void shutdown() { * executed by a specific {@link EntityProcessor} * @throws SQLException */ - public void persistAll(Collection entities, DbGridMetadata identifier) + public void persistAll(Collection entities, DbGridMetadata identifier) throws SQLException { // Extract nested entities and add them to the set of entities Set entitiesToAdd = new HashSet<>(entities); // entities to persist @@ -141,7 +142,7 @@ public void persistAllIgnoreNested( * @param identifier identifier of the grid * @throws SQLException */ - public void persist(C entity, DbGridMetadata identifier) + public void persist(C entity, DbGridMetadata identifier) throws SQLException { if (entity instanceof InputEntity inputEntity) { persistIncludeNested(inputEntity, identifier); @@ -170,7 +171,8 @@ public void persistIgnoreNested(C entity, DbGridMetadata // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - protected void persistListIncludeNested( + /* + protected void persistListIncludeNested( List entities, Class cls, DbGridMetadata identifier) throws SQLException { if (NestedEntity.class.isAssignableFrom(cls)) { entities.forEach( @@ -193,14 +195,16 @@ protected void persistListIncludeNested( } } - public void persistIncludeNested(C entity, DbGridMetadata identifier) + */ + + public void persistIncludeNested(C entity, DbGridMetadata identifier) throws SQLException { Set entitiesToAdd = new HashSet<>(); entitiesToAdd.add(entity); persistAll(entitiesToAdd, identifier); } - private void persistMixedList( + private void persistMixedList( List entities, DbGridMetadata identifier) { Map, List> entitiesPerClass = entities.stream().collect(groupingBy(entity -> (Class) entity.getClass())); @@ -218,7 +222,7 @@ private void persistMixedList( }); } - private , V extends Value> void persistList( + private , V extends Value> void persistList( List entities, Class cls, DbGridMetadata identifier) throws SQLException { // Check if there are only elements of the same class Class firstClass = entities.get(0).getClass(); @@ -226,9 +230,9 @@ private , V extends Value> if (allSameClass) { if (InputEntity.class.isAssignableFrom(cls)) { - insertListIgnoreNested(entities, cls, identifier); + insertListIgnoreNested(entities, cls, identifier, true); } else if (ResultEntity.class.isAssignableFrom(cls)) { - insertListIgnoreNested(entities, cls, identifier); + insertListIgnoreNested(entities, cls, identifier, false); } else if (TimeSeries.class.isAssignableFrom(cls)) { entities.forEach( ts -> { @@ -254,15 +258,14 @@ private , V extends Value> * Writes a list of entities into a sql table. It's necessary that all entities have the same * class. */ - private void insertListIgnoreNested( - List entities, Class cls, DbGridMetadata identifier) throws SQLException { + private void insertListIgnoreNested( + List entities, Class cls, DbGridMetadata identifier, boolean ignoreConflict) throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(cls); String query = basicInsertQueryValuesGrid( schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); - query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier); - System.out.println(query); + query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier, ignoreConflict); connector.executeUpdate(query); } catch (ProcessorProviderException e) { log.error("Exception occurred during processor request: ", e); @@ -336,7 +339,7 @@ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUI SystemParticipants systemParticipants = jointGridContainer.getSystemParticipants(); Set bmPlants = systemParticipants.getBmPlants(); Set chpPlants = systemParticipants.getChpPlants(); - Set evCS = systemParticipants.getEvCS(); + Set evCS = systemParticipants.getEvcs(); Set evs = systemParticipants.getEvs(); Set fixedFeedIns = systemParticipants.getFixedFeedIns(); Set heatPumps = systemParticipants.getHeatPumps(); @@ -344,7 +347,6 @@ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUI Set pvPlants = systemParticipants.getPvPlants(); Set storages = systemParticipants.getStorages(); Set wecPlants = systemParticipants.getWecPlants(); - Set emSystems = systemParticipants.getEmSystems(); // get graphic elements (just for better readability, we could also just get them directly // below) @@ -384,14 +386,13 @@ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUI loads, pvPlants, storages, - wecPlants, - emSystems) + wecPlants) .flatMap(Collection::stream) .map(Extractor::extractOperator) .flatMap(Optional::stream) .collect(Collectors.toSet()); - List toAdd = new LinkedList<>(); + List toAdd = new LinkedList<>(); toAdd.addAll(rawGridElements.allEntitiesAsList()); toAdd.addAll(systemParticipants.allEntitiesAsList()); toAdd.addAll(graphicElements.allEntitiesAsList()); @@ -402,7 +403,7 @@ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUI persistAll(toAdd, identifier); } - private void insert(C entity, DbGridMetadata identifier) + private void insert(C entity, DbGridMetadata identifier) throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); @@ -427,16 +428,24 @@ private void insert(C entity, DbGridMetadata identifier * will be ignored. Conflicts can occur if an entity (e.g. node) already exist. WARNING: It's * assumed that all entities are from the same class C. */ - private String createInsertQueryBodyIgnoreConflict( - List entities, String[] headerElements, DbGridMetadata identifier) + private String createInsertQueryBodyIgnoreConflict( + List entities, String[] headerElements, DbGridMetadata identifier, boolean ignoreConflict) throws ProcessorProviderException { Set> entityFieldData = processorProvider.handleEntities(entities); String queryBody = ""; - queryBody = - queryBody - + entityFieldData.stream() - .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) - .collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + if (ignoreConflict) { + queryBody = + queryBody + + entityFieldData.stream() + .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) + .collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + } else { + queryBody = + queryBody + + entityFieldData.stream() + .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) + .collect(Collectors.joining(",\n", "", ";\n")); + } return queryBody; } @@ -445,7 +454,7 @@ private String createInsertQueryBodyIgnoreConflict( * *

WARNING: It's assumed that all entities are from the same class C. */ - private String createInsertQueryBody( + private String createInsertQueryBody( List entities, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { Set> entityFieldData = processorProvider.handleEntities(entities); @@ -472,7 +481,7 @@ private String queryValueLine( } /** Creates a line with the values of one entity for an insertion query. */ - private String queryValueLine( + private String queryValueLine( C entity, String[] headerElements, DbGridMetadata identifier) throws ProcessorProviderException { return queryValueLine( @@ -510,7 +519,9 @@ private static String basicInsertQuery(String schemaName, String tableName) { /** Provides the insert, column names, grid identifier and the VALUES statement for a query. */ private String basicInsertQueryValuesGrid( String schemaName, String tableName, String[] headerElements) { - String[] addParams = {DbGridMetadata.GRID_UUID}; + String[] addParams = { + DbGridMetadata.GRID_UUID + }; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) @@ -523,7 +534,10 @@ private String basicInsertQueryValuesGrid( */ private String basicInsertQueryValuesITS( String schemaName, String tableName, String[] headerElements) { - String[] addParams = {DbGridMetadata.GRID_UUID, TIME_SERIES}; + String[] addParams = { + DbGridMetadata.GRID_UUID, + TIME_SERIES + }; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) @@ -545,7 +559,7 @@ private String writeOneLine(String[] entries, String[] addParams) { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - public void createClassTable(Class cls, String schemaName) + public void createClassTable(Class cls, String schemaName) throws SQLException, ProcessorProviderException, EntityProcessorException { String query = queryCreateTableUniqueEntity(cls, schemaName); connector.executeUpdate(query); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 08aaef7ec..576e7dc74 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -179,7 +179,6 @@ protected Stream> buildStreamByTableName(String tableName) { protected Stream> executeQuery(String query, AddParams addParams) { try (PreparedStatement ps = connector.getConnection().prepareStatement(query)) { addParams.addParams(ps); - ResultSet resultSet = ps.executeQuery(); return connector.extractFieldMaps(resultSet).stream(); } catch (SQLException e) { @@ -191,4 +190,10 @@ protected Stream> executeQuery(String query, AddParams addPa protected Stream> executeQuery(String query) { return executeQuery(query, x -> {}); } + + public boolean checkExistingTable(String tableName) throws SQLException { + return connector.tableExistsSQL(tableName); + } + + } diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index 5997f65d6..d8860073f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -16,6 +16,7 @@ import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey import edu.ie3.datamodel.io.source.sql.SqlDataSource import edu.ie3.datamodel.models.OperationTime import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.input.EmInput import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.OperatorInput import edu.ie3.datamodel.models.input.connector.LineInput @@ -24,7 +25,6 @@ import edu.ie3.datamodel.models.input.connector.type.LineTypeInput import edu.ie3.datamodel.models.input.connector.type.Transformer2WTypeInput import edu.ie3.datamodel.models.input.graphics.LineGraphicInput import edu.ie3.datamodel.models.input.graphics.NodeGraphicInput -import edu.ie3.datamodel.models.input.system.EmInput import edu.ie3.datamodel.models.input.system.EvcsInput import edu.ie3.datamodel.models.input.system.LoadInput import edu.ie3.datamodel.models.input.system.PvInput @@ -161,15 +161,15 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") Quantity p = Quantities.getQuantity(10, StandardUnits.ACTIVE_POWER_IN) Quantity q = Quantities.getQuantity(10, StandardUnits.REACTIVE_POWER_IN) - PvResult pvResult = new PvResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - WecResult wecResult = new WecResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - EvcsResult evcsResult = new EvcsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) - EmResult emResult = new EmResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, p, q) + PvResult pvResult = new PvResult(TimeUtil.withDefaults.toZonedDateTime("2020-01-30T17:26:44Z"), inputModel, p, q) + WecResult wecResult = new WecResult(TimeUtil.withDefaults.toZonedDateTime("2020-01-30T17:26:44Z"), inputModel, p, q) + EvcsResult evcsResult = new EvcsResult(TimeUtil.withDefaults.toZonedDateTime("2020-01-30T17:26:44Z"), inputModel, p, q) + EmResult emResult = new EmResult(TimeUtil.withDefaults.toZonedDateTime("2020-01-30T17:26:44Z"), inputModel, p, q) Quantity pRef = Quantities.getQuantity(5.1, StandardUnits.ACTIVE_POWER_RESULT) Quantity pMin = Quantities.getQuantity(-6, StandardUnits.ACTIVE_POWER_RESULT) Quantity pMax = Quantities.getQuantity(6, StandardUnits.ACTIVE_POWER_RESULT) - FlexOptionsResult flexOptionsResult = new FlexOptionsResult(uuid, TimeUtil.withDefaults.toZonedDateTime("2020-01-30 17:26:44"), inputModel, pRef, pMin, pMax) + FlexOptionsResult flexOptionsResult = new FlexOptionsResult(TimeUtil.withDefaults.toZonedDateTime("2020-01-30T17:26:44Z"), inputModel, pRef, pMin, pMax) when: sink.persistAll([ @@ -185,8 +185,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri ThermalUnitInputTestData.thermalHouseInput, SystemParticipantTestData.evcsInput, SystemParticipantTestData.loadInput, - SystemParticipantTestData.emInput, - individualTimeSeries + SystemParticipantTestData.emInput ], identifier) then: @@ -206,9 +205,8 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_bus_input", ps -> {}).count() == 1 sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "thermal_house_input", ps -> {}).count() == 1 sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "load_input", ps -> {}).count() == 1 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_input", ps -> {}).count() == 1 + sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "em_input", ps -> {}).count() == 2 sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "ev_res", ps -> {}).count() == 0 - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "time_series_c", ps -> {}).count() == 3 cleanup: sink.shutdown() @@ -264,6 +262,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri OperationTime.notLimited(), Mock(NodeInput), new CosPhiFixed("cosPhiFixed:{(0.0,0.95)}"), + null, 0.2, Quantities.getQuantity(-8.926613807678223, DEGREE_GEOM), Quantities.getQuantity(95d, PERCENT), @@ -309,32 +308,32 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri def "A valid SqlSink throws an exception if a nested entity hasn't all of its nested entity."() { given: - def sink = new SqlSink(schemaName, namingStrategy, connector) + def sink = new SqlSink( + schemaName, + namingStrategy, + connector) + def load = SystemParticipantTestData.loadInput when: - sink.persistIgnoreNested(SystemParticipantTestData.loadInput, identifier) + sink.persistIgnoreNested(load, identifier) then: def exception = thrown(SQLException) exception.message == "ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." - - cleanup: - sink.shutdown() } def "A valid SqlSink can create a table for entity class."() { given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - def wec = SystemParticipantTestData.wecInput + def sink = new SqlSink(schemaName, namingStrategy, connector) + def hp = SystemParticipantTestData.hpInput when: - sink.createClassTable(wec.getClass()) + sink.createClassTable(hp.getClass(), schemaName) then: - sqlSource.executeQuery("SELECT * FROM " + schemaName + "." + "wec_input", ps -> {}).count() == 0 + sqlSource.checkExistingTable("hp_input") } - def "A valid SqlSink throws an exception if a grid does not exist."() { given: def sink = new SqlSink(schemaName, namingStrategy, connector) diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql index 1c3bddf01..0671b350e 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql @@ -38,6 +38,7 @@ CREATE TABLE public.evcs_input ( uuid uuid PRIMARY KEY, charging_points int NOT NULL, + controlling_em uuid, cos_phi_rated TEXT NOT NULL, id TEXT NOT NULL, location_type TEXT NOT NULL, @@ -48,7 +49,7 @@ CREATE TABLE public.evcs_input q_characteristics TEXT NOT NULL, type TEXT NOT NULL, v_2g_support bool NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; @@ -155,6 +156,7 @@ CREATE TABLE public.thermal_bus_input CREATE TABLE public.load_input ( uuid uuid PRIMARY KEY, + controlling_em uuid NOT NULL, cos_phi_rated TEXT NOT NULL, dsm bool NOT NULL, e_cons_annual double precision NOT NULL, @@ -166,7 +168,7 @@ CREATE TABLE public.load_input operator uuid, q_characteristics TEXT NOT NULL, s_rated double precision NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; @@ -174,15 +176,13 @@ CREATE TABLE public.load_input CREATE TABLE public.em_input ( uuid uuid PRIMARY KEY, - connected_assets TEXT NOT NULL, control_strategy TEXT NOT NULL, + controlling_em uuid, id TEXT NOT NULL, - node uuid NOT NULL, operates_from timestamp with time zone, operates_until timestamp with time zone, operator uuid, - q_characteristics TEXT NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; @@ -192,6 +192,7 @@ CREATE TABLE public.pv_input uuid uuid PRIMARY KEY, albedo double precision NOT NULL, azimuth double precision NOT NULL, + controlling_em uuid, cos_phi_rated TEXT NOT NULL, elevation_angle double precision NOT NULL, eta_conv double precision NOT NULL, @@ -205,7 +206,7 @@ CREATE TABLE public.pv_input operator uuid, q_characteristics TEXT NOT NULL, s_rated double precision NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; @@ -213,6 +214,7 @@ CREATE TABLE public.pv_input CREATE TABLE public.storage_input ( uuid uuid PRIMARY KEY, + controlling_em uuid NOT NULL, id TEXT NOT NULL, node uuid NOT NULL, operates_from timestamp with time zone, @@ -220,7 +222,7 @@ CREATE TABLE public.storage_input operator uuid, q_characteristics TEXT NOT NULL, type uuid NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql index 5f3268340..3ef2160f7 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql @@ -1,6 +1,5 @@ CREATE TABLE public.load_profile_g2 ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, day_of_week TEXT NOT NULL, quarter_hour_of_day TEXT NOT NULL, diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql index 924d8efb1..208f1f655 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql @@ -1,6 +1,5 @@ CREATE TABLE public.pv_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p double precision NOT NULL, q double precision NOT NULL, @@ -12,7 +11,6 @@ CREATE TABLE public.pv_res CREATE TABLE public.wec_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p double precision NOT NULL, q double precision NOT NULL, @@ -24,7 +22,6 @@ CREATE TABLE public.wec_res CREATE TABLE public.ev_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p double precision NOT NULL, q double precision NOT NULL, @@ -37,7 +34,6 @@ CREATE TABLE public.ev_res CREATE TABLE public.evcs_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p double precision NOT NULL, q double precision NOT NULL, @@ -49,7 +45,6 @@ CREATE TABLE public.evcs_res CREATE TABLE public.em_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p double precision NOT NULL, q double precision NOT NULL, @@ -61,7 +56,6 @@ CREATE TABLE public.em_res CREATE TABLE public.flex_options_res ( - uuid uuid PRIMARY KEY, input_model uuid NOT NULL, p_max double precision NOT NULL, p_min double precision NOT NULL, diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql index d77ad1895..be58fda6a 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql @@ -1,6 +1,5 @@ CREATE TABLE public.time_series_c ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, price double precision, @@ -18,7 +17,6 @@ CREATE UNIQUE INDEX time_series_c_series_time ON time_series_c USING btree (time CREATE TABLE public.time_series_p ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, @@ -33,7 +31,6 @@ CREATE UNIQUE INDEX time_series_p_series_time ON time_series_p USING btree (time CREATE TABLE public.time_series_pq ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, @@ -49,7 +46,6 @@ CREATE UNIQUE INDEX time_series_pq_series_time ON time_series_pq USING btree (ti CREATE TABLE public.time_series_h ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, heat_demand double precision, @@ -64,7 +60,6 @@ CREATE UNIQUE INDEX time_series_h_series_time ON time_series_h USING btree (time CREATE TABLE public.time_series_ph ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, @@ -80,7 +75,6 @@ CREATE UNIQUE INDEX time_series_ph_series_time ON time_series_ph USING btree (ti CREATE TABLE public.time_series_pqh ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, time timestamp with time zone NOT NULL, p double precision, @@ -97,7 +91,6 @@ CREATE UNIQUE INDEX time_series_pqh_series_time ON time_series_pqh USING btree ( CREATE TABLE public.time_series_weather ( - uuid uuid PRIMARY KEY, time_series uuid NOT NULL, coordinate TEXT NOT NULL, time timestamp with time zone NOT NULL, diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql index 3c4579882..cddf9f995 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql @@ -42,17 +42,13 @@ CREATE TABLE public.storage_type_input active_power_gradient double precision NOT NULL, capex double precision NOT NULL, cos_phi_rated TEXT NOT NULL, - dod double precision NOT NULL, e_storage double precision NOT NULL, eta double precision NOT NULL, id TEXT NOT NULL, - life_cycle double precision NOT NULL, - life_time double precision NOT NULL, opex double precision NOT NULL, p_max double precision NOT NULL, s_rated double precision NOT NULL, - - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; From 15d97ccb0f2590fd42b93dfc029106fbda2a3c6b Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Mon, 1 Jul 2024 08:06:51 +0200 Subject: [PATCH 10/19] replace UniqueEntity --- src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 59fe40c1f..06510927d 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -164,7 +164,7 @@ public void persist(C entity, DbGridMetadata identifier) * @param identifier identifier of the grid * @throws SQLException */ - public void persistIgnoreNested(C entity, DbGridMetadata identifier) + public void persistIgnoreNested(C entity, DbGridMetadata identifier) throws SQLException { insert(entity, identifier); } From d77e478f2bb81212eae906dc5dd7e23d6c879ebb Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Mon, 1 Jul 2024 08:07:35 +0200 Subject: [PATCH 11/19] Main Method --- src/main/java/edu/ie3/datamodel/Main.java | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/java/edu/ie3/datamodel/Main.java diff --git a/src/main/java/edu/ie3/datamodel/Main.java b/src/main/java/edu/ie3/datamodel/Main.java new file mode 100644 index 000000000..f1daddcc6 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/Main.java @@ -0,0 +1,21 @@ +package edu.ie3.datamodel; + +import edu.ie3.datamodel.exceptions.EntityProcessorException; +import edu.ie3.datamodel.exceptions.ProcessorProviderException; +import edu.ie3.datamodel.io.SqlUtils; +import edu.ie3.datamodel.models.input.system.type.StorageTypeInput; + +public class Main { + public static void main(String[] args) { + try { + System.out.println(SqlUtils.queryCreateTableUniqueEntity( + StorageTypeInput.class, + "public" + )); + } catch (EntityProcessorException e) { + throw new RuntimeException(e); + } catch (ProcessorProviderException e) { + throw new RuntimeException(e); + } + } +} From 6667c3d0e8a1d9861c5b821511bb2597bc5d9076 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Tue, 2 Jul 2024 11:51:49 +0200 Subject: [PATCH 12/19] Code Cleaning --- src/main/java/edu/ie3/datamodel/Main.java | 24 +-- .../edu/ie3/datamodel/io/DbGridMetadata.java | 3 +- .../java/edu/ie3/datamodel/io/IoUtil.java | 14 -- .../java/edu/ie3/datamodel/io/SqlUtils.java | 11 +- .../datamodel/io/connectors/SqlConnector.java | 16 +- .../io/naming/DatabaseNamingStrategy.java | 2 +- .../ie3/datamodel/io/processor/Processor.java | 1 - .../io/processor/ProcessorProvider.java | 4 +- .../edu/ie3/datamodel/io/sink/SqlSink.java | 169 +++++++----------- .../io/source/sql/SqlDataSource.java | 2 - .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 30 ++-- .../ie3/test/common/TimeSeriesTestData.groovy | 1 - .../datamodel/io/source/sql/_types/types.sql | 2 +- 13 files changed, 112 insertions(+), 167 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/Main.java b/src/main/java/edu/ie3/datamodel/Main.java index f1daddcc6..28224b428 100644 --- a/src/main/java/edu/ie3/datamodel/Main.java +++ b/src/main/java/edu/ie3/datamodel/Main.java @@ -1,3 +1,8 @@ +/* + * © 2024. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ package edu.ie3.datamodel; import edu.ie3.datamodel.exceptions.EntityProcessorException; @@ -6,16 +11,13 @@ import edu.ie3.datamodel.models.input.system.type.StorageTypeInput; public class Main { - public static void main(String[] args) { - try { - System.out.println(SqlUtils.queryCreateTableUniqueEntity( - StorageTypeInput.class, - "public" - )); - } catch (EntityProcessorException e) { - throw new RuntimeException(e); - } catch (ProcessorProviderException e) { - throw new RuntimeException(e); - } + public static void main(String[] args) { + try { + System.out.println(SqlUtils.queryCreateTableUniqueEntity(StorageTypeInput.class, "public")); + } catch (EntityProcessorException e) { + throw new RuntimeException(e); + } catch (ProcessorProviderException e) { + throw new RuntimeException(e); } + } } diff --git a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java index 29df2bd90..2a2d0d72c 100644 --- a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java +++ b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io; -import static edu.ie3.datamodel.io.IoUtil.quote; +import static edu.ie3.datamodel.io.SqlUtils.quote; import java.util.UUID; import java.util.stream.Stream; @@ -21,6 +21,7 @@ public String toString() { return GRID_NAME + "=" + gridName + ", " + GRID_UUID + "=" + uuid.toString(); } + /** @return Stream with grid uuid */ public Stream getStreamForQuery() { return Stream.of(quote(uuid.toString(), "'")); } diff --git a/src/main/java/edu/ie3/datamodel/io/IoUtil.java b/src/main/java/edu/ie3/datamodel/io/IoUtil.java index 9f9f12a74..0968b4f28 100644 --- a/src/main/java/edu/ie3/datamodel/io/IoUtil.java +++ b/src/main/java/edu/ie3/datamodel/io/IoUtil.java @@ -7,8 +7,6 @@ import java.io.File; import java.nio.file.Path; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.Optional; public class IoUtil { @@ -51,16 +49,4 @@ public static Path harmonizeFileSeparator(Path path) { public static Optional pathOption(String in) { return Optional.of(Path.of(in)); } - - public String timeFormatter(ZonedDateTime time, String timePattern) { - return time.format(DateTimeFormatter.ofPattern(timePattern)); - } - - public static String quote(String input, String quoteSymbol) { - if (input == "") { - return "NULL"; - } else { - return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; - } - } } diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 1c40edea4..df884258b 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -12,10 +12,10 @@ import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; import edu.ie3.datamodel.io.processor.ProcessorProvider; import edu.ie3.datamodel.models.Entity; -import edu.ie3.datamodel.models.UniqueEntity; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; import org.slf4j.Logger; @@ -43,8 +43,7 @@ public static String queryCreateGridTable(String schemaName) { } /** @return query to create a SQL table for an unique entity */ - public static String queryCreateTableUniqueEntity( - Class cls, String schemaName) + public static String queryCreateTableUniqueEntity(Class cls, String schemaName) throws EntityProcessorException, ProcessorProviderException { ProcessorProvider processorProvider = new ProcessorProvider(); DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); @@ -75,7 +74,7 @@ public static String queryCreateTableUniqueEntity( * @return Map column name -> data type */ public static Map columnToSqlDataType() { - HashMap map = new HashMap(); + Map map = new HashMap<>(); map.put("uuid", "uuid PRIMARY KEY"); map.put("time_series", "uuid NOT NULL"); @@ -181,7 +180,7 @@ public static Map columnToSqlDataType() { map.put("opex", "double precision NOT NULL"); map.put("active_power_gradient", "double precision NOT NULL"); - // not all data types are implemented + // TODO: Not all data types are implemented return map; } @@ -192,7 +191,7 @@ public static Map columnToSqlDataType() { * @return input with quoteSymbol */ public static String quote(String input, String quoteSymbol) { - if (input == "") { + if (Objects.equals(input, "")) { return "NULL"; } else { return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index f43ff8f8b..b25d751ef 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -156,11 +156,19 @@ public Map extractFieldMap(ResultSet rs) { return insensitiveFieldsToAttributes; } + /** + * Executes a query to check if a table exists + * + * @param tableName Name of the table, that should be checked + * @return True, if the table exists + */ public boolean tableExistsSQL(String tableName) throws SQLException { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT count(*) " - + "FROM information_schema.tables " - + "WHERE table_name = ?" - + "LIMIT 1;"); + PreparedStatement preparedStatement = + connection.prepareStatement( + "SELECT count(*) " + + "FROM information_schema.tables " + + "WHERE table_name = ?" + + "LIMIT 1;"); preparedStatement.setString(1, tableName); ResultSet resultSet = preparedStatement.executeQuery(); diff --git a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java index 5e85512f8..b3b35172b 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/DatabaseNamingStrategy.java @@ -8,12 +8,12 @@ import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput; import edu.ie3.datamodel.models.value.Value; -import edu.ie3.datamodel.models.Entity; import java.util.Optional; /** A naming strategy for database entities */ diff --git a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java index e8cce1ba7..faf79daa8 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/Processor.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/Processor.java @@ -24,7 +24,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; import javax.measure.Quantity; diff --git a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java index 7064107f7..58cad7354 100644 --- a/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java +++ b/src/main/java/edu/ie3/datamodel/io/processor/ProcessorProvider.java @@ -78,8 +78,8 @@ Try, ProcessorProviderException> handleEntity(T en .transformF(ProcessorProviderException::new)); } - public Set> handleEntities( - List entities) throws ProcessorProviderException { + public Set> handleEntities(List entities) + throws ProcessorProviderException { Set setOfEntities = new HashSet<>(entities); Set> setOfMaps = new HashSet<>(); for (T entity : setOfEntities) { diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 06510927d..7d9e224a1 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -18,7 +18,6 @@ import edu.ie3.datamodel.io.processor.ProcessorProvider; import edu.ie3.datamodel.io.processor.timeseries.TimeSeriesProcessorKey; import edu.ie3.datamodel.models.Entity; -import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.input.*; import edu.ie3.datamodel.models.input.connector.*; import edu.ie3.datamodel.models.input.container.GraphicElements; @@ -83,28 +82,25 @@ public void shutdown() { * @param identifier identifier of the grid * @param bounded to be all unique entities. Handling of specific entities is normally then * executed by a specific {@link EntityProcessor} - * @throws SQLException */ - public void persistAll(Collection entities, DbGridMetadata identifier) - throws SQLException { + public void persistAll(Collection entities, DbGridMetadata identifier) { // Extract nested entities and add them to the set of entities Set entitiesToAdd = new HashSet<>(entities); // entities to persist - entities.stream() - .forEach( - entity -> { - if (entity instanceof NestedEntity nestedEntity) { - try { - entitiesToAdd.addAll( - (List) Extractor.extractElements(nestedEntity).stream().toList()); - } catch (ExtractorException e) { - log.error( - String.format( - "An error occurred during extraction of nested entity'%s': ", - entity.getClass()), - e); - } - } - }); + entities.forEach( + entity -> { + if (entity instanceof NestedEntity nestedEntity) { + try { + entitiesToAdd.addAll( + (List) Extractor.extractElements(nestedEntity).stream().toList()); + } catch (ExtractorException e) { + log.error( + String.format( + "An error occurred during extraction of nested entity'%s': ", + entity.getClass()), + e); + } + } + }); // Persist the entities in hierarchic order to avoid failure because of foreign keys for (Class cls : hierarchicInsert()) { @@ -116,24 +112,12 @@ public void persistAll(Collection entities, DbGridMetadata entitiesToAdd.removeIf( ent -> cls.isAssignableFrom( - ent.getClass())); // maybe it's not necessary but I'm not sure if there are + ent.getClass())); // maybe it's not necessary, but I'm not sure if there are // entities who aren't in the hierarchic structure } persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities } - /** - * Persist multiple input entities in a collection. In contrast to {@link SqlSink#persistAll} this - * function does not extract nested entities. - * - * @param entities a collection of entities that should be persisted - * @param identifier identifier of the grid - */ - public void persistAllIgnoreNested( - Collection entities, DbGridMetadata identifier) { - persistMixedList(new ArrayList<>(entities), identifier); - } - /** * Persist an entity. By default this method take care about the extraction process of nested * entities (if any) @@ -142,8 +126,7 @@ public void persistAllIgnoreNested( * @param identifier identifier of the grid * @throws SQLException */ - public void persist(C entity, DbGridMetadata identifier) - throws SQLException { + public void persist(C entity, DbGridMetadata identifier) throws SQLException { if (entity instanceof InputEntity inputEntity) { persistIncludeNested(inputEntity, identifier); } else if (entity instanceof ResultEntity resultEntity) { @@ -162,50 +145,29 @@ public void persist(C entity, DbGridMetadata identifier) * * @param entity the entity that should be persisted * @param identifier identifier of the grid - * @throws SQLException */ public void persistIgnoreNested(C entity, DbGridMetadata identifier) throws SQLException { insert(entity, identifier); } - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - /* - protected void persistListIncludeNested( - List entities, Class cls, DbGridMetadata identifier) throws SQLException { - if (NestedEntity.class.isAssignableFrom(cls)) { - entities.forEach( - entity -> { - try { - List arr = - new ArrayList<>(Extractor.extractElements((NestedEntity) entity)); - persistAll(arr, identifier); - } catch (ExtractorException | SQLException e) { - log.error( - String.format( - "An error occurred during extraction of nested entity'%s': ", - cls.getSimpleName()), - e); - } - }); - insertListIgnoreNested(entities, cls, identifier); - } else { - insertListIgnoreNested(entities, cls, identifier); - } - } - + /** + * Persist an entity and all nested entities. + * + * @param entity the entity that should be persisted + * @param identifier identifier of the grid */ - - public void persistIncludeNested(C entity, DbGridMetadata identifier) - throws SQLException { + public void persistIncludeNested(C entity, DbGridMetadata identifier) { Set entitiesToAdd = new HashSet<>(); entitiesToAdd.add(entity); persistAll(entitiesToAdd, identifier); } - private void persistMixedList( - List entities, DbGridMetadata identifier) { + /** + * Persist a list of entities with different types. To minimize the number of queries, the + * entities will be grouped by their class. + */ + private void persistMixedList(List entities, DbGridMetadata identifier) { Map, List> entitiesPerClass = entities.stream().collect(groupingBy(entity -> (Class) entity.getClass())); entitiesPerClass.forEach( @@ -222,6 +184,10 @@ private void persistMixedList( }); } + /** + * Persist a list of entities with same types. To minimize the number of queries, the entities + * will be grouped by their class. + */ private , V extends Value> void persistList( List entities, Class cls, DbGridMetadata identifier) throws SQLException { // Check if there are only elements of the same class @@ -259,19 +225,24 @@ private , V extends Value> void p * class. */ private void insertListIgnoreNested( - List entities, Class cls, DbGridMetadata identifier, boolean ignoreConflict) throws SQLException { + List entities, Class cls, DbGridMetadata identifier, boolean ignoreConflict) + throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(cls); String query = basicInsertQueryValuesGrid( schemaName, databaseNamingStrategy.getEntityName(cls).orElseThrow(), headerElements); - query = query + createInsertQueryBodyIgnoreConflict(entities, headerElements, identifier, ignoreConflict); + query = + query + + createInsertQueryBodyIgnoreConflict( + entities, headerElements, identifier, ignoreConflict); connector.executeUpdate(query); } catch (ProcessorProviderException e) { log.error("Exception occurred during processor request: ", e); } } + /** Persist one time series. */ protected , V extends Value> void persistTimeSeries( TimeSeries timeSeries, DbGridMetadata identifier) throws SQLException { try { @@ -288,7 +259,7 @@ protected , V extends Value> void persistTimeSeries private , V extends Value> void persistTimeSeries( TimeSeries timeSeries, String[] headerElements, DbGridMetadata identifier) - throws ProcessorProviderException, IOException, SQLException { + throws ProcessorProviderException, IOException { try { String query = basicInsertQueryValuesITS( @@ -322,8 +293,8 @@ private , V extends Value> void persistTimeSeries( } } - public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) - throws SQLException { + /** Persists a whole {@link JointGridContainer}. */ + public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) { DbGridMetadata identifier = new DbGridMetadata(jointGridContainer.getGridName(), gridUUID); // get raw grid entities with types or operators @@ -399,12 +370,11 @@ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUI toAdd.addAll(types); toAdd.addAll(operators); - // persist all entities persistAll(toAdd, identifier); } - private void insert(C entity, DbGridMetadata identifier) - throws SQLException { + /** Executes a query to insert a single entity to a SQL database. */ + private void insert(C entity, DbGridMetadata identifier) throws SQLException { try { String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); String query = @@ -432,38 +402,18 @@ private String createInsertQueryBodyIgnoreConflict( List entities, String[] headerElements, DbGridMetadata identifier, boolean ignoreConflict) throws ProcessorProviderException { Set> entityFieldData = processorProvider.handleEntities(entities); - String queryBody = ""; + String suffix; if (ignoreConflict) { - queryBody = - queryBody - + entityFieldData.stream() - .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) - .collect(Collectors.joining(",\n", "", "\nON CONFLICT (uuid) DO NOTHING;")); + suffix = "\nON CONFLICT (uuid) DO NOTHING;"; } else { - queryBody = - queryBody - + entityFieldData.stream() - .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) - .collect(Collectors.joining(",\n", "", ";\n")); + suffix = ";\n"; } - return queryBody; - } - - /** - * Provides the value lists for an insertion query. - * - *

WARNING: It's assumed that all entities are from the same class C. - */ - private String createInsertQueryBody( - List entities, String[] headerElements, DbGridMetadata identifier) - throws ProcessorProviderException { - Set> entityFieldData = processorProvider.handleEntities(entities); String queryBody = ""; queryBody = queryBody + entityFieldData.stream() .map(data -> queryValueLine(sqlEntityFieldData(data), headerElements, identifier)) - .collect(Collectors.joining(",\n", "", ";")); + .collect(Collectors.joining(",\n", "", suffix)); return queryBody; } @@ -519,9 +469,7 @@ private static String basicInsertQuery(String schemaName, String tableName) { /** Provides the insert, column names, grid identifier and the VALUES statement for a query. */ private String basicInsertQueryValuesGrid( String schemaName, String tableName, String[] headerElements) { - String[] addParams = { - DbGridMetadata.GRID_UUID - }; + String[] addParams = {DbGridMetadata.GRID_UUID}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) @@ -534,10 +482,7 @@ private String basicInsertQueryValuesGrid( */ private String basicInsertQueryValuesITS( String schemaName, String tableName, String[] headerElements) { - String[] addParams = { - DbGridMetadata.GRID_UUID, - TIME_SERIES - }; + String[] addParams = {DbGridMetadata.GRID_UUID, TIME_SERIES}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) @@ -559,12 +504,23 @@ private String writeOneLine(String[] entries, String[] addParams) { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + /** + * Creates a table for an entity class + * + * @param cls The class for which a table should be created + * @param schemaName Schema name of the database + */ public void createClassTable(Class cls, String schemaName) throws SQLException, ProcessorProviderException, EntityProcessorException { String query = queryCreateTableUniqueEntity(cls, schemaName); connector.executeUpdate(query); } + /** + * Creates a table for a grid + * + * @param schemaName Schema name of the database + */ public void createGridTable(String schemaName) throws SQLException { String query = queryCreateGridTable(schemaName); connector.executeUpdate(query); @@ -581,7 +537,6 @@ private static List> hierarchicInsert() { sortedInsert.add(ConnectorInput.class); // 6a. ConnectorInput sortedInsert.add(SystemParticipantInput.class); // 6b. SystemParticipantInput sortedInsert.add(GraphicInput.class); // 7. GraphicInput - // 8. Rest return sortedInsert; } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 576e7dc74..6db92daa0 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -194,6 +194,4 @@ protected Stream> executeQuery(String query) { public boolean checkExistingTable(String tableName) throws SQLException { return connector.tableExistsSQL(tableName); } - - } diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index d8860073f..d0f0a5487 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -5,6 +5,10 @@ */ package edu.ie3.datamodel.io.sink +import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM +import static edu.ie3.util.quantities.PowerSystemUnits.KILOVOLTAMPERE +import static tech.units.indriya.unit.Units.PERCENT + import edu.ie3.datamodel.io.DbGridMetadata import edu.ie3.datamodel.io.connectors.SqlConnector import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy @@ -46,29 +50,23 @@ import edu.ie3.datamodel.models.value.EnergyPriceValue import edu.ie3.datamodel.models.value.Value import edu.ie3.test.common.GridTestData import edu.ie3.test.common.SampleJointGrid +import edu.ie3.test.common.SystemParticipantTestData import edu.ie3.test.common.ThermalUnitInputTestData import edu.ie3.test.common.TimeSeriesTestData import edu.ie3.test.helper.TestContainerHelper import edu.ie3.util.TimeUtil import org.testcontainers.containers.Container import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.spock.Testcontainers import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification -import org.testcontainers.spock.Testcontainers import tech.units.indriya.quantity.Quantities +import java.sql.SQLException import javax.measure.Quantity import javax.measure.quantity.Power -import edu.ie3.test.common.SystemParticipantTestData - -import java.sql.SQLException - -import static edu.ie3.util.quantities.PowerSystemUnits.DEGREE_GEOM -import static edu.ie3.util.quantities.PowerSystemUnits.KILOVOLTAMPERE -import static tech.units.indriya.unit.Units.PERCENT - @Testcontainers class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeriesTestData { @@ -309,9 +307,9 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri def "A valid SqlSink throws an exception if a nested entity hasn't all of its nested entity."() { given: def sink = new SqlSink( - schemaName, - namingStrategy, - connector) + schemaName, + namingStrategy, + connector) def load = SystemParticipantTestData.loadInput when: @@ -325,14 +323,14 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri def "A valid SqlSink can create a table for entity class."() { given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - def hp = SystemParticipantTestData.hpInput + def sink = new SqlSink(schemaName, namingStrategy, connector) + def hp = SystemParticipantTestData.hpInput when: - sink.createClassTable(hp.getClass(), schemaName) + sink.createClassTable(hp.getClass(), schemaName) then: - sqlSource.checkExistingTable("hp_input") + sqlSource.checkExistingTable("hp_input") } def "A valid SqlSink throws an exception if a grid does not exist."() { given: diff --git a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy index 001bf35b9..18e5030d3 100644 --- a/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/TimeSeriesTestData.groovy @@ -11,7 +11,6 @@ import static tech.units.indriya.unit.Units.METRE_PER_SECOND import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation -import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile import edu.ie3.datamodel.models.timeseries.IntValue diff --git a/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql b/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql index 56251bb93..13bb195bf 100644 --- a/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql +++ b/src/test/resources/edu/ie3/datamodel/io/source/sql/_types/types.sql @@ -1,6 +1,6 @@ CREATE TABLE public.line_type_input ( - uuid BINARY(16) NOT NULL, + uuid uuid NOT NULL, id varchar NOT NULL, v_rated double precision NOT NULL, i_max double precision NOT NULL, From 6bb4abe329fa42d7f73a5754028dda862a662abb Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Tue, 2 Jul 2024 14:49:07 +0200 Subject: [PATCH 13/19] Code Cleaning --- src/main/java/edu/ie3/datamodel/Main.java | 23 --- .../java/edu/ie3/datamodel/io/SqlUtils.java | 155 ------------------ .../edu/ie3/datamodel/io/sink/SqlSink.java | 22 --- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 11 -- 4 files changed, 211 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/Main.java diff --git a/src/main/java/edu/ie3/datamodel/Main.java b/src/main/java/edu/ie3/datamodel/Main.java deleted file mode 100644 index 28224b428..000000000 --- a/src/main/java/edu/ie3/datamodel/Main.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * © 2024. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel; - -import edu.ie3.datamodel.exceptions.EntityProcessorException; -import edu.ie3.datamodel.exceptions.ProcessorProviderException; -import edu.ie3.datamodel.io.SqlUtils; -import edu.ie3.datamodel.models.input.system.type.StorageTypeInput; - -public class Main { - public static void main(String[] args) { - try { - System.out.println(SqlUtils.queryCreateTableUniqueEntity(StorageTypeInput.class, "public")); - } catch (EntityProcessorException e) { - throw new RuntimeException(e); - } catch (ProcessorProviderException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index df884258b..79f286eb5 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -5,19 +5,7 @@ */ package edu.ie3.datamodel.io; -import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; - -import edu.ie3.datamodel.exceptions.EntityProcessorException; -import edu.ie3.datamodel.exceptions.ProcessorProviderException; -import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; -import edu.ie3.datamodel.io.processor.ProcessorProvider; -import edu.ie3.datamodel.models.Entity; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,149 +30,6 @@ public static String queryCreateGridTable(String schemaName) { + endQueryCreateTable; } - /** @return query to create a SQL table for an unique entity */ - public static String queryCreateTableUniqueEntity(Class cls, String schemaName) - throws EntityProcessorException, ProcessorProviderException { - ProcessorProvider processorProvider = new ProcessorProvider(); - DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); - String[] headerElements = processorProvider.getHeaderElements(cls); - Stream strHeader = - Stream.concat(Arrays.stream(headerElements), Stream.of(DbGridMetadata.GRID_UUID)); - Stream dtHeader = - strHeader.map( - element -> - camelCaseToSnakeCase(element) - + " " - + columnToSqlDataType().get(camelCaseToSnakeCase(element))); - return "CREATE TABLE " - + schemaName - + "." - + namingStrategy.getEntityName(cls).orElseThrow() - + "\n(\n\t" - + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) - + "\n)\n\t" - + "WITHOUT OIDS\n" - + "\t" - + "TABLESPACE pg_default;\n"; - } - - /** - * Map to create a SQL table for an entity with the right data types. - * - * @return Map column name -> data type - */ - public static Map columnToSqlDataType() { - Map map = new HashMap<>(); - - map.put("uuid", "uuid PRIMARY KEY"); - map.put("time_series", "uuid NOT NULL"); - map.put("time", "timestamp with time zone NOT NULL"); - map.put("p", "double precision NOT NULL"); - map.put("q", "double precision NOT NULL"); - map.put("c", "double precision NOT NULL"); - map.put("s_rated", "double precision NOT NULL"); - - map.put("cost_controlled", "bool NOT NULL"); - map.put("feed_in_tariff", "int NOT NULL"); - map.put("id", "TEXT NOT NULL"); - map.put("market_reaction", "bool NOT NULL"); - map.put("node", "uuid NOT NULL"); - map.put("operates_from", "timestamp with time zone"); - map.put("operates_until", "timestamp with time zone"); - map.put("operator", "uuid"); - map.put("q_characteristics", "TEXT NOT NULL"); - map.put("geo_position", "TEXT NOT NULL"); - map.put("length", "double precision NOT NULL"); - map.put("node_a", "uuid NOT NULL"); - map.put("node_b", "uuid NOT NULL"); - map.put("olm_characteristic", "TEXT NOT NULL"); - map.put("parallel_devices", "int NOT NULL"); - map.put("cos_phi_rated", "TEXT NOT NULL"); - map.put("dsm", "bool NOT NULL"); - map.put("e_cons_annual", "double precision NOT NULL"); - map.put("load_profile", "TEXT NOT NULL"); - map.put("controlling_em", "uuid NOT NULL"); - - map.put("auto_tap", "bool NOT NULL"); - map.put("tap_pos", "int NOT NULL"); - map.put("type", "uuid NOT NULL"); - - map.put("v_ang", "bool NOT NULL"); - map.put("v_mag", "bool NOT NULL"); - map.put("slack", "bool NOT NULL"); - map.put("subnet", "int NOT NULL"); - map.put("v_rated", "double precision NOT NULL"); - map.put("v_target", "double precision NOT NULL"); - map.put("volt_lvl", "TEXT NOT NULL"); - map.put("charging_points", "int NOT NULL"); - map.put("location_type", "TEXT NOT NULL"); - map.put("v_2g_support", "bool NOT NULL"); - - map.put("albedo", "double precision NOT NULL"); - map.put("azimuth", "double precision NOT NULL"); - map.put("elevation_angle", "double precision NOT NULL"); - map.put("eta_conv", "double precision NOT NULL"); - map.put("k_g", "double precision NOT NULL"); - map.put("k_t", "double precision NOT NULL"); - - map.put("grid_name", "TEXT NOT NULL"); - map.put("grid_uuid", "uuid NOT NULL"); - - map.put("b_m", "double precision NOT NULL"); - map.put("d_phi", "double precision NOT NULL"); - map.put("d_v", "double precision NOT NULL"); - map.put("g_m", "double precision NOT NULL"); - map.put("r_sc", "double precision NOT NULL"); - map.put("tap_max", "int NOT NULL"); - map.put("tap_min", "int NOT NULL"); - map.put("tap_neutr", "int NOT NULL"); - map.put("tap_side", "bool NOT NULL"); - map.put("v_rated_a", "int NOT NULL"); - map.put("v_rated_b", "int NOT NULL"); - map.put("x_sc", "int NOT NULL"); - map.put("graphic_layer", "TEXT NOT NULL"); - map.put("line", "uuid NOT NULL"); - map.put("path", "TEXT NOT NULL"); - map.put("point", "TEXT NOT NULL"); - map.put("inlet_temp", "double precision NOT NULL"); - map.put("return_temp", "double precision NOT NULL"); - map.put("storage_volume_lvl", "double precision NOT NULL"); - map.put("storage_volume_lvl_min", "double precision NOT NULL"); - map.put("thermal_bus", "uuid NOT NULL"); - map.put("eth_capa", "double precision NOT NULL"); - map.put("eth_losses", "double precision NOT NULL"); - map.put("lower_temperature_limit", "double precision NOT NULL"); - map.put("target_temperature", "double precision NOT NULL"); - map.put("upper_temperature_limit", "double precision NOT NULL"); - map.put("b", "double precision NOT NULL"); - map.put("g", "double precision NOT NULL"); - map.put("i_max", "double precision NOT NULL"); - map.put("r", "double precision NOT NULL"); - map.put("x", "double precision NOT NULL"); - - map.put("connected_assets", "TEXT NOT NULL"); - map.put("capex", "double precision NOT NULL"); - map.put("control_strategy", "TEXT NOT NULL"); - - map.put("input_model", "uuid NOT NULL"); - map.put("soc", "double precision NOT NULL"); - map.put("p_max", "double precision NOT NULL"); - map.put("p_min", "double precision NOT NULL"); - map.put("p_ref", "double precision NOT NULL"); - - map.put("dod", "double precision NOT NULL"); - map.put("e_storage", "double precision NOT NULL"); - map.put("eta", "double precision NOT NULL"); - map.put("life_cycle", "double precision NOT NULL"); - map.put("life_time", "double precision NOT NULL"); - map.put("opex", "double precision NOT NULL"); - map.put("active_power_gradient", "double precision NOT NULL"); - - // TODO: Not all data types are implemented - - return map; - } - /** * To avoid data type conflicts while insertion into a SQL table all columns should be quoted. * diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 7d9e224a1..d153ef12f 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -504,28 +504,6 @@ private String writeOneLine(String[] entries, String[] addParams) { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - /** - * Creates a table for an entity class - * - * @param cls The class for which a table should be created - * @param schemaName Schema name of the database - */ - public void createClassTable(Class cls, String schemaName) - throws SQLException, ProcessorProviderException, EntityProcessorException { - String query = queryCreateTableUniqueEntity(cls, schemaName); - connector.executeUpdate(query); - } - - /** - * Creates a table for a grid - * - * @param schemaName Schema name of the database - */ - public void createGridTable(String schemaName) throws SQLException { - String query = queryCreateGridTable(schemaName); - connector.executeUpdate(query); - } - /** @return insertion order for unique entities */ private static List> hierarchicInsert() { List> sortedInsert = new ArrayList<>(); diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index d0f0a5487..3147ba2ab 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -321,17 +321,6 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." } - def "A valid SqlSink can create a table for entity class."() { - given: - def sink = new SqlSink(schemaName, namingStrategy, connector) - def hp = SystemParticipantTestData.hpInput - - when: - sink.createClassTable(hp.getClass(), schemaName) - - then: - sqlSource.checkExistingTable("hp_input") - } def "A valid SqlSink throws an exception if a grid does not exist."() { given: def sink = new SqlSink(schemaName, namingStrategy, connector) From f62a10d1cd663defd9e43b90b32cadd6456ed8cd Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Tue, 2 Jul 2024 15:31:23 +0200 Subject: [PATCH 14/19] SQL Caps --- .../edu/ie3/datamodel/io/sink/_sql/grids.sql | 2 +- .../datamodel/io/sink/_sql/input_entities.sql | 216 +++++++++--------- .../datamodel/io/sink/_sql/load_profile.sql | 6 +- .../io/sink/_sql/result_entities.sql | 64 +++--- .../datamodel/io/sink/_sql/sql_test_data.sql | 8 +- .../datamodel/io/sink/_sql/time_series.sql | 72 +++--- .../edu/ie3/datamodel/io/sink/_sql/types.sql | 64 +++--- 7 files changed, 216 insertions(+), 216 deletions(-) diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql index 466aad358..4ba8069a0 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/grids.sql @@ -1,6 +1,6 @@ CREATE TABLE public.grids ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, name TEXT NOT NULL ) WITHOUT OIDS diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql index 0671b350e..f53c01c27 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql @@ -1,228 +1,228 @@ CREATE TABLE public.node_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, geo_position TEXT NOT NULL, id TEXT NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, - slack bool NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, + slack BOOL NOT NULL, subnet int NOT NULL, - v_rated double precision NOT NULL, - v_target double precision NOT NULL, + v_rated DOUBLE PRECISION NOT NULL, + v_target DOUBLE PRECISION NOT NULL, volt_lvl TEXT NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.transformer_2_w_input ( - uuid uuid PRIMARY KEY, - auto_tap bool NOT NULL, + uuid UUID PRIMARY KEY, + auto_tap BOOL NOT NULL, id TEXT NOT NULL, - node_a uuid NOT NULL REFERENCES node_input(uuid), - node_b uuid NOT NULL REFERENCES node_input(uuid), - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + node_a UUID NOT NULL REFERENCES node_input(uuid), + node_b UUID NOT NULL REFERENCES node_input(uuid), + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, parallel_devices int NOT NULL, tap_pos int NOT NULL, - type uuid NOT NULL REFERENCES transformer_2_w_type_input(uuid), - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + type UUID NOT NULL REFERENCES transformer_2_w_type_input(uuid), + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.evcs_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, charging_points int NOT NULL, - controlling_em uuid, + controlling_em UUID, cos_phi_rated TEXT NOT NULL, id TEXT NOT NULL, location_type TEXT NOT NULL, - node uuid NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + node UUID NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, q_characteristics TEXT NOT NULL, type TEXT NOT NULL, - v_2g_support bool NOT NULL, - grid_uuid uuid NOT NULL + v_2g_support BOOL NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.line_graphic_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, graphic_layer TEXT NOT NULL, - line uuid NOT NULL, + line UUID NOT NULL, path TEXT, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.node_graphic_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, graphic_layer TEXT NOT NULL, - node uuid NOT NULL, + node UUID NOT NULL, path TEXT, point TEXT NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.cylindrical_storage_input ( - uuid uuid PRIMARY KEY, - c double precision NOT NULL, + uuid UUID PRIMARY KEY, + c DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - inlet_temp double precision NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, - return_temp double precision NOT NULL, - storage_volume_lvl double precision NOT NULL, - storage_volume_lvl_min double precision NOT NULL, - thermal_bus uuid NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + inlet_temp DOUBLE PRECISION NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, + return_temp DOUBLE PRECISION NOT NULL, + storage_volume_lvl DOUBLE PRECISION NOT NULL, + storage_volume_lvl_min DOUBLE PRECISION NOT NULL, + thermal_bus UUID NOT NULL, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.thermal_house_input ( - uuid uuid PRIMARY KEY, - eth_capa double precision NOT NULL, - eth_losses double precision NOT NULL, + uuid UUID PRIMARY KEY, + eth_capa DOUBLE PRECISION NOT NULL, + eth_losses DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - lower_temperature_limit double precision NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, - target_temperature double precision NOT NULL, - thermal_bus uuid NOT NULL, - upper_temperature_limit double precision NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + lower_temperature_limit DOUBLE PRECISION NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, + target_temperature DOUBLE PRECISION NOT NULL, + thermal_bus UUID NOT NULL, + upper_temperature_limit DOUBLE PRECISION NOT NULL, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.operator_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, id TEXT NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.line_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, geo_position TEXT NOT NULL, id TEXT NOT NULL, - length double precision NOT NULL, - node_a uuid NOT NULL, - node_b uuid NOT NULL, + length DOUBLE PRECISION NOT NULL, + node_a UUID NOT NULL, + node_b UUID NOT NULL, olm_characteristic TEXT NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, parallel_devices int NOT NULL, - type uuid NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + type UUID NOT NULL, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.thermal_bus_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, id TEXT NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.load_input ( - uuid uuid PRIMARY KEY, - controlling_em uuid NOT NULL, + uuid UUID PRIMARY KEY, + controlling_em UUID NOT NULL, cos_phi_rated TEXT NOT NULL, - dsm bool NOT NULL, - e_cons_annual double precision NOT NULL, + dsm BOOL NOT NULL, + e_cons_annual DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, load_profile TEXT NOT NULL, - node uuid NOT NULL REFERENCES node_input(uuid), - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + node UUID NOT NULL REFERENCES node_input(uuid), + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, q_characteristics TEXT NOT NULL, - s_rated double precision NOT NULL, - grid_uuid uuid NOT NULL + s_rated DOUBLE PRECISION NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.em_input ( - uuid uuid PRIMARY KEY, + uuid UUID PRIMARY KEY, control_strategy TEXT NOT NULL, - controlling_em uuid, + controlling_em UUID, id TEXT NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, - grid_uuid uuid NOT NULL + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.pv_input ( - uuid uuid PRIMARY KEY, - albedo double precision NOT NULL, - azimuth double precision NOT NULL, - controlling_em uuid, + uuid UUID PRIMARY KEY, + albedo DOUBLE PRECISION NOT NULL, + azimuth DOUBLE PRECISION NOT NULL, + controlling_em UUID, cos_phi_rated TEXT NOT NULL, - elevation_angle double precision NOT NULL, - eta_conv double precision NOT NULL, + elevation_angle DOUBLE PRECISION NOT NULL, + eta_conv DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - k_g double precision NOT NULL, - k_t double precision NOT NULL, - market_reaction bool NOT NULL, - node uuid NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + k_g DOUBLE PRECISION NOT NULL, + k_t DOUBLE PRECISION NOT NULL, + market_reaction BOOL NOT NULL, + node UUID NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, q_characteristics TEXT NOT NULL, - s_rated double precision NOT NULL, - grid_uuid uuid NOT NULL + s_rated DOUBLE PRECISION NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.storage_input ( - uuid uuid PRIMARY KEY, - controlling_em uuid NOT NULL, + uuid UUID PRIMARY KEY, + controlling_em UUID NOT NULL, id TEXT NOT NULL, - node uuid NOT NULL, - operates_from timestamp with time zone, - operates_until timestamp with time zone, - operator uuid, + node UUID NOT NULL, + operates_from TIMESTAMP WITH TIME ZONE, + operates_until TIMESTAMP WITH TIME ZONE, + operator UUID, q_characteristics TEXT NOT NULL, - type uuid NOT NULL, - grid_uuid uuid NOT NULL + type UUID NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; \ No newline at end of file diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql index 3ef2160f7..474be75de 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/load_profile.sql @@ -1,10 +1,10 @@ CREATE TABLE public.load_profile_g2 ( - time_series uuid NOT NULL, + time_series UUID NOT NULL, day_of_week TEXT NOT NULL, quarter_hour_of_day TEXT NOT NULL, - p double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + p DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql index 208f1f655..a924203fc 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/result_entities.sql @@ -1,67 +1,67 @@ CREATE TABLE public.pv_res ( - input_model uuid NOT NULL, - p double precision NOT NULL, - q double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p DOUBLE PRECISION NOT NULL, + q DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.wec_res ( - input_model uuid NOT NULL, - p double precision NOT NULL, - q double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p DOUBLE PRECISION NOT NULL, + q DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.ev_res ( - input_model uuid NOT NULL, - p double precision NOT NULL, - q double precision NOT NULL, - soc double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p DOUBLE PRECISION NOT NULL, + q DOUBLE PRECISION NOT NULL, + soc DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.evcs_res ( - input_model uuid NOT NULL, - p double precision NOT NULL, - q double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p DOUBLE PRECISION NOT NULL, + q DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.em_res ( - input_model uuid NOT NULL, - p double precision NOT NULL, - q double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p DOUBLE PRECISION NOT NULL, + q DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.flex_options_res ( - input_model uuid NOT NULL, - p_max double precision NOT NULL, - p_min double precision NOT NULL, - p_ref double precision NOT NULL, - time timestamp with time zone NOT NULL, - grid_uuid uuid NOT NULL + input_model UUID NOT NULL, + p_max DOUBLE PRECISION NOT NULL, + p_min DOUBLE PRECISION NOT NULL, + p_ref DOUBLE PRECISION NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql index a5cb071e5..5a329811c 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql @@ -1,9 +1,9 @@ CREATE TABLE public.test_data_sql ( - uuid uuid PRIMARY KEY, - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - p double precision NOT NULL + uuid UUID PRIMARY KEY, + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + p DOUBLE PRECISION NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql index be58fda6a..cdceb0e52 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/time_series.sql @@ -1,9 +1,9 @@ CREATE TABLE public.time_series_c ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - price double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + price DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -17,10 +17,10 @@ CREATE UNIQUE INDEX time_series_c_series_time ON time_series_c USING btree (time CREATE TABLE public.time_series_p ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - p double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + p DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -31,11 +31,11 @@ CREATE UNIQUE INDEX time_series_p_series_time ON time_series_p USING btree (time CREATE TABLE public.time_series_pq ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - p double precision, - q double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + p DOUBLE PRECISION, + q DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -46,10 +46,10 @@ CREATE UNIQUE INDEX time_series_pq_series_time ON time_series_pq USING btree (ti CREATE TABLE public.time_series_h ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - heat_demand double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + heat_demand DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -60,11 +60,11 @@ CREATE UNIQUE INDEX time_series_h_series_time ON time_series_h USING btree (time CREATE TABLE public.time_series_ph ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - p double precision, - heat_demand double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + p DOUBLE PRECISION, + heat_demand DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -75,12 +75,12 @@ CREATE UNIQUE INDEX time_series_ph_series_time ON time_series_ph USING btree (ti CREATE TABLE public.time_series_pqh ( - time_series uuid NOT NULL, - time timestamp with time zone NOT NULL, - p double precision, - q double precision, - heat_demand double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time_series UUID NOT NULL, + time TIMESTAMP WITH TIME ZONE NOT NULL, + p DOUBLE PRECISION, + q DOUBLE PRECISION, + heat_demand DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; @@ -91,15 +91,15 @@ CREATE UNIQUE INDEX time_series_pqh_series_time ON time_series_pqh USING btree ( CREATE TABLE public.time_series_weather ( - time_series uuid NOT NULL, + time_series UUID NOT NULL, coordinate TEXT NOT NULL, - time timestamp with time zone NOT NULL, - diffuse_irradiance double precision, - direct_irradiance double precision, - direction double precision, - temperature double precision, - velocity double precision, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + time TIMESTAMP WITH TIME ZONE NOT NULL, + diffuse_irradiance DOUBLE PRECISION, + direct_irradiance DOUBLE PRECISION, + direction DOUBLE PRECISION, + temperature DOUBLE PRECISION, + velocity DOUBLE PRECISION, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql index cddf9f995..03efd5342 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/types.sql @@ -1,54 +1,54 @@ CREATE TABLE public.transformer_2_w_type_input ( - uuid uuid PRIMARY KEY, - b_m double precision NOT NULL, - d_phi double precision NOT NULL, - d_v double precision NOT NULL, - g_m double precision NOT NULL, + uuid UUID PRIMARY KEY, + b_m DOUBLE PRECISION NOT NULL, + d_phi DOUBLE PRECISION NOT NULL, + d_v DOUBLE PRECISION NOT NULL, + g_m DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - r_sc double precision NOT NULL, - s_rated double precision NOT NULL, - tap_max int NOT NULL, - tap_min int NOT NULL, - tap_neutr int NOT NULL, - tap_side bool NOT NULL, - v_rated_a double precision NOT NULL, - v_rated_b double precision NOT NULL, - x_sc double precision NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + r_sc DOUBLE PRECISION NOT NULL, + s_rated DOUBLE PRECISION NOT NULL, + tap_max INT NOT NULL, + tap_min INT NOT NULL, + tap_neutr INT NOT NULL, + tap_side BOOL NOT NULL, + v_rated_a DOUBLE PRECISION NOT NULL, + v_rated_b DOUBLE PRECISION NOT NULL, + x_sc DOUBLE PRECISION NOT NULL, + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.line_type_input ( - uuid uuid PRIMARY KEY, - b double precision NOT NULL, - g double precision NOT NULL, - i_max double precision NOT NULL, + uuid UUID PRIMARY KEY, + b DOUBLE PRECISION NOT NULL, + g DOUBLE PRECISION NOT NULL, + i_max DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - r double precision NOT NULL, - v_rated double precision NOT NULL, - x double precision NOT NULL, + r DOUBLE PRECISION NOT NULL, + v_rated DOUBLE PRECISION NOT NULL, + x DOUBLE PRECISION NOT NULL, - grid_uuid uuid NOT NULL REFERENCES grids(uuid) + grid_uuid UUID NOT NULL REFERENCES grids(uuid) ) WITHOUT OIDS TABLESPACE pg_default; CREATE TABLE public.storage_type_input ( - uuid uuid PRIMARY KEY, - active_power_gradient double precision NOT NULL, - capex double precision NOT NULL, + uuid UUID PRIMARY KEY, + active_power_gradient DOUBLE PRECISION NOT NULL, + capex DOUBLE PRECISION NOT NULL, cos_phi_rated TEXT NOT NULL, - e_storage double precision NOT NULL, - eta double precision NOT NULL, + e_storage DOUBLE PRECISION NOT NULL, + eta DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - opex double precision NOT NULL, - p_max double precision NOT NULL, - s_rated double precision NOT NULL, - grid_uuid uuid NOT NULL + opex DOUBLE PRECISION NOT NULL, + p_max DOUBLE PRECISION NOT NULL, + s_rated DOUBLE PRECISION NOT NULL, + grid_uuid UUID NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; From 5d0a594c4a4502de0009b6d0d586211eff28b314 Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Wed, 3 Jul 2024 12:42:31 +0200 Subject: [PATCH 15/19] Fixing sonarqube --- .../java/edu/ie3/datamodel/io/SqlUtils.java | 4 +- .../datamodel/io/connectors/SqlConnector.java | 37 ++++++++++++------- .../edu/ie3/datamodel/io/sink/SqlSink.java | 21 ++--------- .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 6 +-- src/test/resources/log4j2-test.xml | 2 +- 5 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 79f286eb5..880a459ae 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -13,7 +13,7 @@ public class SqlUtils { protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); private static final String endQueryCreateTable = - ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; + ")\n \t WITHOUT OIDS\n \t TABLESPACE pg_default;"; private SqlUtils() { throw new IllegalStateException("Utility classes cannot be instantiated"); @@ -36,7 +36,7 @@ public static String queryCreateGridTable(String schemaName) { * @return input with quoteSymbol */ public static String quote(String input, String quoteSymbol) { - if (Objects.equals(input, "")) { + if (Objects.equals(input, "") || Objects.equals(input, "null")) { return "NULL"; } else { return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index b25d751ef..e16d69a6b 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -61,11 +61,16 @@ public ResultSet executeQuery(Statement stmt, String query) throws SQLException /** * Executes an update query * - * @param updateQuery the query to execute + * @param query the query to execute * @return The number of updates or a negative number if the execution failed */ - public int executeUpdate(String updateQuery) throws SQLException { - return getConnection().createStatement().executeUpdate(updateQuery); + public int executeUpdate(String query) throws SQLException { + try (Statement statement = getConnection().createStatement()) { + return statement.executeUpdate(query); + } catch (SQLException e) { + throw new SQLException( + String.format("Error at execution of query, SQLReason: '%s'", e.getMessage()), e); + } } /** @@ -162,17 +167,21 @@ public Map extractFieldMap(ResultSet rs) { * @param tableName Name of the table, that should be checked * @return True, if the table exists */ - public boolean tableExistsSQL(String tableName) throws SQLException { - PreparedStatement preparedStatement = - connection.prepareStatement( - "SELECT count(*) " - + "FROM information_schema.tables " - + "WHERE table_name = ?" - + "LIMIT 1;"); - preparedStatement.setString(1, tableName); + public boolean tableExistsSQL(String tableName) { + String query = + "SELECT count(*) " + + "FROM information_schema.tables " + + "WHERE table_name = ?" + + "LIMIT 1;"; + try (PreparedStatement ps = getConnection().prepareStatement(query)) { + ps.setString(1, tableName); - ResultSet resultSet = preparedStatement.executeQuery(); - resultSet.next(); - return resultSet.getInt(1) != 0; + ResultSet resultSet = ps.executeQuery(); + resultSet.next(); + return resultSet.getInt(1) != 0; + } catch (SQLException e) { + log.error("Error during execution of query {}", query, e); + } + return false; } } diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index d153ef12f..f150e0065 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -33,7 +33,6 @@ import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; import edu.ie3.util.StringUtils; -import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.stream.Collectors; @@ -105,9 +104,7 @@ public void persistAll(Collection entities, DbGridMetadata // Persist the entities in hierarchic order to avoid failure because of foreign keys for (Class cls : hierarchicInsert()) { persistMixedList( - entitiesToAdd.stream() - .filter(ent -> cls.isAssignableFrom(ent.getClass())) - .collect(Collectors.toList()), + entitiesToAdd.stream().filter(ent -> cls.isAssignableFrom(ent.getClass())).toList(), identifier); entitiesToAdd.removeIf( ent -> @@ -202,15 +199,7 @@ private , V extends Value> void p } else if (TimeSeries.class.isAssignableFrom(cls)) { entities.forEach( ts -> { - try { - persistTimeSeries((TimeSeries) ts, identifier); - } catch (SQLException e) { - throw new RuntimeException( - String.format( - "An error occurred during extraction of entity '%s', SQLReason: '%s'", - cls.getSimpleName(), e.getMessage()), - e); - } + persistTimeSeries((TimeSeries) ts, identifier); }); } else { log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); @@ -244,7 +233,7 @@ private void insertListIgnoreNested( /** Persist one time series. */ protected , V extends Value> void persistTimeSeries( - TimeSeries timeSeries, DbGridMetadata identifier) throws SQLException { + TimeSeries timeSeries, DbGridMetadata identifier) { try { TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); String[] headerElements = processorProvider.getHeaderElements(key); @@ -252,14 +241,12 @@ protected , V extends Value> void persistTimeSeries } catch (ProcessorProviderException e) { log.error( "Exception occurred during receiving of header elements. Cannot write this element.", e); - } catch (IOException e) { - log.error("Exception occurred during closing of writer.", e); } } private , V extends Value> void persistTimeSeries( TimeSeries timeSeries, String[] headerElements, DbGridMetadata identifier) - throws ProcessorProviderException, IOException { + throws ProcessorProviderException { try { String query = basicInsertQueryValuesITS( diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index 3147ba2ab..0e64df0dc 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -277,7 +277,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri then: def exception = thrown(SQLException) - exception.message.contains("ERROR: invalid input syntax for type uuid: \"null\"\n") + exception.message.contains("ERROR: null value in column \"node\" of relation \"pv_input\" violates not-null constraint") cleanup: sink.shutdown() @@ -317,8 +317,8 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri then: def exception = thrown(SQLException) - exception.message == "ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + - " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\"." + exception.message.contains("ERROR: insert or update on table \"load_input\" violates foreign key constraint \"load_input_node_fkey\"\n" + + " Detail: Key (node)=(4ca90220-74c2-4369-9afa-a18bf068840d) is not present in table \"node_input\".") } def "A valid SqlSink throws an exception if a grid does not exist."() { diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index dd7c536aa..fd2d36637 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -25,7 +25,7 @@ - + From 4da951e1e47230d5ae2d93a4185305a53ed3b91e Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Wed, 3 Jul 2024 13:51:51 +0200 Subject: [PATCH 16/19] optimize tests --- src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index 0e64df0dc..13be399c5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -128,7 +128,6 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri TimeSeriesProcessorKey timeSeriesProcessorKey = new TimeSeriesProcessorKey(IndividualTimeSeries, TimeBasedValue, EnergyPriceValue) HashMap timeSeriesProcessorMap = new HashMap<>() timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) - IndividualTimeSeries individualTimeSeries = individualEnergyPriceTimeSeries SqlSink sink = new SqlSink(schemaName, new ProcessorProvider([ @@ -155,7 +154,6 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri ], timeSeriesProcessorMap), namingStrategy, connector) - UUID uuid = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") Quantity p = Quantities.getQuantity(10, StandardUnits.ACTIVE_POWER_IN) Quantity q = Quantities.getQuantity(10, StandardUnits.REACTIVE_POWER_IN) From 62564e25035498e5555caac9fd757952c8c4c6dd Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Fri, 30 Aug 2024 09:06:35 +0200 Subject: [PATCH 17/19] changes after review --- .../edu/ie3/datamodel/io/DbGridMetadata.java | 8 +++---- .../java/edu/ie3/datamodel/io/SqlUtils.java | 2 +- .../edu/ie3/datamodel/io/sink/SqlSink.java | 12 ++++------ .../ie3/datamodel/io/sink/SqlSinkTest.groovy | 23 +------------------ .../datamodel/io/sink/_sql/input_entities.sql | 19 ++++++++------- .../datamodel/io/sink/_sql/sql_test_data.sql | 16 ------------- 6 files changed, 19 insertions(+), 61 deletions(-) delete mode 100644 src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql diff --git a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java index 2a2d0d72c..b613f0894 100644 --- a/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java +++ b/src/main/java/edu/ie3/datamodel/io/DbGridMetadata.java @@ -13,12 +13,12 @@ /** Class for identification of entities and results from grids in SQL databases. */ public record DbGridMetadata(String gridName, UUID uuid) { - public static final String GRID_TABLE = "grids"; - public static final String GRID_NAME = "grid_name"; - public static final String GRID_UUID = "grid_uuid"; + public static final String GRID_TABLE_COLUMN = "grids"; + public static final String GRID_NAME_COLUMN = "grid_name"; + public static final String GRID_UUID_COLUMN = "grid_uuid"; public String toString() { - return GRID_NAME + "=" + gridName + ", " + GRID_UUID + "=" + uuid.toString(); + return GRID_NAME_COLUMN + "=" + gridName + ", " + GRID_UUID_COLUMN + "=" + uuid.toString(); } /** @return Stream with grid uuid */ diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 880a459ae..0fd51d529 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -25,7 +25,7 @@ private static String beginQueryCreateTable(String schemaName, String tableName) /** @return query to create a SQL table for a grid */ public static String queryCreateGridTable(String schemaName) { - return beginQueryCreateTable(schemaName, DbGridMetadata.GRID_TABLE) + return beginQueryCreateTable(schemaName, DbGridMetadata.GRID_TABLE_COLUMN) + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" + endQueryCreateTable; } diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index f150e0065..3345f8509 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -106,17 +106,13 @@ public void persistAll(Collection entities, DbGridMetadata persistMixedList( entitiesToAdd.stream().filter(ent -> cls.isAssignableFrom(ent.getClass())).toList(), identifier); - entitiesToAdd.removeIf( - ent -> - cls.isAssignableFrom( - ent.getClass())); // maybe it's not necessary, but I'm not sure if there are - // entities who aren't in the hierarchic structure + entitiesToAdd.removeIf(ent -> cls.isAssignableFrom(ent.getClass())); } persistMixedList(new ArrayList<>(entitiesToAdd), identifier); // persist left entities } /** - * Persist an entity. By default this method take care about the extraction process of nested + * Persist an entity. By default this method takes care of the extraction process of nested * entities (if any) * * @param entity the entity that should be persisted @@ -456,7 +452,7 @@ private static String basicInsertQuery(String schemaName, String tableName) { /** Provides the insert, column names, grid identifier and the VALUES statement for a query. */ private String basicInsertQueryValuesGrid( String schemaName, String tableName, String[] headerElements) { - String[] addParams = {DbGridMetadata.GRID_UUID}; + String[] addParams = {DbGridMetadata.GRID_UUID_COLUMN}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) @@ -469,7 +465,7 @@ private String basicInsertQueryValuesGrid( */ private String basicInsertQueryValuesITS( String schemaName, String tableName, String[] headerElements) { - String[] addParams = {DbGridMetadata.GRID_UUID, TIME_SERIES}; + String[] addParams = {DbGridMetadata.GRID_UUID_COLUMN, TIME_SERIES}; return basicInsertQuery(schemaName, tableName) + " " + writeOneLine(StringUtils.camelCaseToSnakeCase(headerElements), addParams) diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy index 13be399c5..9e45cf059 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/SqlSinkTest.groovy @@ -130,28 +130,7 @@ class SqlSinkTest extends Specification implements TestContainerHelper, TimeSeri timeSeriesProcessorMap.put(timeSeriesProcessorKey, timeSeriesProcessor) SqlSink sink = new SqlSink(schemaName, - new ProcessorProvider([ - new ResultEntityProcessor(PvResult), - new ResultEntityProcessor(WecResult), - new ResultEntityProcessor(EvResult), - new ResultEntityProcessor(EvcsResult), - new ResultEntityProcessor(EmResult), - new ResultEntityProcessor(FlexOptionsResult), - new InputEntityProcessor(Transformer2WInput), - new InputEntityProcessor(NodeInput), - new InputEntityProcessor(EvcsInput), - new InputEntityProcessor(Transformer2WTypeInput), - new InputEntityProcessor(LineGraphicInput), - new InputEntityProcessor(NodeGraphicInput), - new InputEntityProcessor(CylindricalStorageInput), - new InputEntityProcessor(ThermalHouseInput), - new InputEntityProcessor(OperatorInput), - new InputEntityProcessor(LineInput), - new InputEntityProcessor(ThermalBusInput), - new InputEntityProcessor(LineTypeInput), - new InputEntityProcessor(LoadInput), - new InputEntityProcessor(EmInput) - ], timeSeriesProcessorMap), + new ProcessorProvider(), namingStrategy, connector) UUID inputModel = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql index f53c01c27..9b63f3fba 100644 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql +++ b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/input_entities.sql @@ -79,18 +79,17 @@ CREATE TABLE public.node_graphic_input CREATE TABLE public.cylindrical_storage_input ( - uuid UUID PRIMARY KEY, + uuid uuid PRIMARY KEY, c DOUBLE PRECISION NOT NULL, id TEXT NOT NULL, - inlet_temp DOUBLE PRECISION NOT NULL, - operates_from TIMESTAMP WITH TIME ZONE, - operates_until TIMESTAMP WITH TIME ZONE, - operator UUID, - return_temp DOUBLE PRECISION NOT NULL, - storage_volume_lvl DOUBLE PRECISION NOT NULL, - storage_volume_lvl_min DOUBLE PRECISION NOT NULL, - thermal_bus UUID NOT NULL, - grid_uuid UUID NOT NULL REFERENCES grids(uuid) + inlet_temp double precision NOT NULL, + operates_from timestamp with time zone, + operates_until timestamp with time zone, + operator uuid, + return_temp double precision NOT NULL, + storage_volume_lvl double precision NOT NULL, + thermal_bus uuid NOT NULL, + grid_uuid uuid NOT NULL ) WITHOUT OIDS TABLESPACE pg_default; diff --git a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql b/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql deleted file mode 100644 index 5a329811c..000000000 --- a/src/test/resources/edu/ie3/datamodel/io/sink/_sql/sql_test_data.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE public.test_data_sql -( - uuid UUID PRIMARY KEY, - time_series UUID NOT NULL, - time TIMESTAMP WITH TIME ZONE NOT NULL, - p DOUBLE PRECISION NOT NULL -) - WITHOUT OIDS - TABLESPACE pg_default; - -CREATE INDEX test_data_sql_series_id ON test_data_sql USING hash (time_series); - --- Order of columns is important when using btree: https://www.postgresql.org/docs/14/indexes-multicolumn.html --- Column time_series needs to placed as the first argument since we at most use an equality constraint on --- time_series and a range query on time. -CREATE UNIQUE INDEX test_data_sql_series_time ON test_data_sql USING btree (time_series, time); \ No newline at end of file From 802d4d242cdb5665638ea24a7cb9dcf95fec69ac Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Fri, 30 Aug 2024 09:42:58 +0200 Subject: [PATCH 18/19] fixing Sonarqube --- .../java/edu/ie3/datamodel/io/SqlUtils.java | 4 +-- .../datamodel/io/connectors/SqlConnector.java | 24 -------------- .../edu/ie3/datamodel/io/sink/SqlSink.java | 31 ++++++++++--------- .../io/source/sql/SqlDataSource.java | 4 --- 4 files changed, 18 insertions(+), 45 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java index 0fd51d529..501c8c6aa 100644 --- a/src/main/java/edu/ie3/datamodel/io/SqlUtils.java +++ b/src/main/java/edu/ie3/datamodel/io/SqlUtils.java @@ -12,7 +12,7 @@ public class SqlUtils { protected static final Logger log = LoggerFactory.getLogger(SqlUtils.class); - private static final String endQueryCreateTable = + private static final String END_QUERY_CREATE_TABLE = ")\n \t WITHOUT OIDS\n \t TABLESPACE pg_default;"; private SqlUtils() { @@ -27,7 +27,7 @@ private static String beginQueryCreateTable(String schemaName, String tableName) public static String queryCreateGridTable(String schemaName) { return beginQueryCreateTable(schemaName, DbGridMetadata.GRID_TABLE_COLUMN) + "\tuuid uuid PRIMARY KEY,\n\tname TEXT NOT NULL\n" - + endQueryCreateTable; + + END_QUERY_CREATE_TABLE; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java index e16d69a6b..b1e463180 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java @@ -160,28 +160,4 @@ public Map extractFieldMap(ResultSet rs) { } return insensitiveFieldsToAttributes; } - - /** - * Executes a query to check if a table exists - * - * @param tableName Name of the table, that should be checked - * @return True, if the table exists - */ - public boolean tableExistsSQL(String tableName) { - String query = - "SELECT count(*) " - + "FROM information_schema.tables " - + "WHERE table_name = ?" - + "LIMIT 1;"; - try (PreparedStatement ps = getConnection().prepareStatement(query)) { - ps.setString(1, tableName); - - ResultSet resultSet = ps.executeQuery(); - resultSet.next(); - return resultSet.getInt(1) != 0; - } catch (SQLException e) { - log.error("Error during execution of query {}", query, e); - } - return false; - } } diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 3345f8509..06e4c646c 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -193,10 +193,7 @@ private , V extends Value> void p } else if (ResultEntity.class.isAssignableFrom(cls)) { insertListIgnoreNested(entities, cls, identifier, false); } else if (TimeSeries.class.isAssignableFrom(cls)) { - entities.forEach( - ts -> { - persistTimeSeries((TimeSeries) ts, identifier); - }); + entities.forEach(ts -> persistTimeSeries((TimeSeries) ts, identifier)); } else { log.error("I don't know how to handle an entity of class {}", cls.getSimpleName()); } @@ -262,20 +259,24 @@ private , V extends Value> void persistTimeSeries( identifier, timeSeries.getUuid().toString())) .collect(Collectors.joining(",\n", "", ";")); - try { - connector.executeUpdate(query); - } catch (SQLException e) { - throw new RuntimeException( - String.format( - "An error occurred during extraction of the time series, SQLReason: '%s'", - e.getMessage()), - e); - } + executeQueryToPersist(query); } catch (ProcessorProviderException e) { throw new ProcessorProviderException("Exception occurred during processor request: ", e); } } + private void executeQueryToPersist(String query) { + try { + connector.executeUpdate(query); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "An error occurred during extraction of the time series, SQLReason: '%s'", + e.getMessage()), + e); + } + } + /** Persists a whole {@link JointGridContainer}. */ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) { DbGridMetadata identifier = new DbGridMetadata(jointGridContainer.getGridName(), gridUUID); @@ -427,13 +428,13 @@ private String queryTimeSeriesValueLine( Map entityFieldData, String[] headerElements, DbGridMetadata identifier, - String TSuuid) { + String tsUuid) { return writeOneLine( Stream.concat( Stream.concat( Arrays.stream(headerElements).map(entityFieldData::get), identifier.getStreamForQuery()), - Stream.of(quote(TSuuid, "'")))); + Stream.of(quote(tsUuid, "'")))); } private LinkedHashMap sqlEntityFieldData( diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java index 6db92daa0..7fafca42a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java @@ -190,8 +190,4 @@ protected Stream> executeQuery(String query, AddParams addPa protected Stream> executeQuery(String query) { return executeQuery(query, x -> {}); } - - public boolean checkExistingTable(String tableName) throws SQLException { - return connector.tableExistsSQL(tableName); - } } From 07eed44031f1ed60cc08e4ed3e77a30a114c64fd Mon Sep 17 00:00:00 2001 From: Johannes Bao Date: Tue, 29 Oct 2024 13:48:07 +0100 Subject: [PATCH 19/19] simplify persist joint grid --- .../edu/ie3/datamodel/io/sink/SqlSink.java | 78 +------------------ 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java index 06e4c646c..907322b1f 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/SqlSink.java @@ -20,10 +20,7 @@ import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.input.*; import edu.ie3.datamodel.models.input.connector.*; -import edu.ie3.datamodel.models.input.container.GraphicElements; import edu.ie3.datamodel.models.input.container.JointGridContainer; -import edu.ie3.datamodel.models.input.container.RawGridElements; -import edu.ie3.datamodel.models.input.container.SystemParticipants; import edu.ie3.datamodel.models.input.graphics.GraphicInput; import edu.ie3.datamodel.models.input.system.*; import edu.ie3.datamodel.models.input.thermal.ThermalBusInput; @@ -280,80 +277,7 @@ private void executeQueryToPersist(String query) { /** Persists a whole {@link JointGridContainer}. */ public void persistJointGrid(JointGridContainer jointGridContainer, UUID gridUUID) { DbGridMetadata identifier = new DbGridMetadata(jointGridContainer.getGridName(), gridUUID); - - // get raw grid entities with types or operators - RawGridElements rawGridElements = jointGridContainer.getRawGrid(); - Set nodes = rawGridElements.getNodes(); - Set lines = rawGridElements.getLines(); - Set transformer2Ws = rawGridElements.getTransformer2Ws(); - Set transformer3Ws = rawGridElements.getTransformer3Ws(); - Set switches = rawGridElements.getSwitches(); - Set measurementUnits = rawGridElements.getMeasurementUnits(); - - // get system participants with types or operators - SystemParticipants systemParticipants = jointGridContainer.getSystemParticipants(); - Set bmPlants = systemParticipants.getBmPlants(); - Set chpPlants = systemParticipants.getChpPlants(); - Set evCS = systemParticipants.getEvcs(); - Set evs = systemParticipants.getEvs(); - Set fixedFeedIns = systemParticipants.getFixedFeedIns(); - Set heatPumps = systemParticipants.getHeatPumps(); - Set loads = systemParticipants.getLoads(); - Set pvPlants = systemParticipants.getPvPlants(); - Set storages = systemParticipants.getStorages(); - Set wecPlants = systemParticipants.getWecPlants(); - - // get graphic elements (just for better readability, we could also just get them directly - // below) - GraphicElements graphicElements = jointGridContainer.getGraphics(); - - // extract types - Set types = - Stream.of( - lines, - transformer2Ws, - transformer3Ws, - bmPlants, - chpPlants, - evs, - heatPumps, - storages, - wecPlants) - .flatMap(Collection::stream) - .map(Extractor::extractType) - .collect(Collectors.toSet()); - - // extract operators - Set operators = - Stream.of( - nodes, - lines, - transformer2Ws, - transformer3Ws, - switches, - measurementUnits, - bmPlants, - chpPlants, - evCS, - evs, - fixedFeedIns, - heatPumps, - loads, - pvPlants, - storages, - wecPlants) - .flatMap(Collection::stream) - .map(Extractor::extractOperator) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - - List toAdd = new LinkedList<>(); - toAdd.addAll(rawGridElements.allEntitiesAsList()); - toAdd.addAll(systemParticipants.allEntitiesAsList()); - toAdd.addAll(graphicElements.allEntitiesAsList()); - toAdd.addAll(types); - toAdd.addAll(operators); - + List toAdd = new LinkedList<>(jointGridContainer.allEntitiesAsList()); persistAll(toAdd, identifier); }