Improving column name validation

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Changing from comparing strings to comparing uuids in `EntitySource.findFirstEntityByUuid` [#829](https://github.com/ie3-institute/PowerSystemDataModel/issues/829)
 - Adding JavaDoc to `EntitySource.safeMapGet` [#828](https://github.com/ie3-institute/PowerSystemDataModel/issues/828)
 - Abstracting some methods in `ValidationUtils` [#852](https://github.com/ie3-institute/PowerSystemDataModel/issues/852)
+- Enhancing the error message for coordinate sources with invalid column names [#670](https://github.com/ie3-institute/PowerSystemDataModel/issues/670) 
+- Allowing for additional unused columns in sources [#839](https://github.com/ie3-institute/PowerSystemDataModel/issues/839)
+- Improving column name validation to only run once per source [#849](https://github.com/ie3-institute/PowerSystemDataModel/issues/849)
 
 ## [4.1.0] - 2023-11-02
 

src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java

@@ -5,7 +5,7 @@
 */
 package edu.ie3.datamodel.exceptions;
 
-/** Is thrown, when an something went wrong during entity creation process in a EntityFactory */
+/** Is thrown, when something went wrong during entity creation process in a EntityFactory */
 public class FactoryException extends RuntimeException {
   public FactoryException(final String message, final Throwable cause) {
     super(message, cause);

src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java

@@ -10,11 +10,16 @@
 import com.couchbase.client.java.Cluster;
 import com.couchbase.client.java.ClusterOptions;
 import com.couchbase.client.java.Collection;
+import com.couchbase.client.java.json.JsonObject;
 import com.couchbase.client.java.kv.GetResult;
 import com.couchbase.client.java.kv.MutationResult;
 import com.couchbase.client.java.query.QueryResult;
+import edu.ie3.datamodel.io.source.SourceValidator;
+import edu.ie3.datamodel.models.value.WeatherValue;
 import java.time.Duration;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -56,6 +61,34 @@ public CouchbaseConnector(
     cluster = Cluster.connect(url, clusterOptions);
   }
 
+  /**
+   * This method should be used to validate a given couchbaseDb.
+   *
+   * @param entityClass class of the entity
+   * @param validator for validation
+   */
+  @SuppressWarnings("unchecked")
+  public final void validateDb(
+      Class<WeatherValue> entityClass, SourceValidator<WeatherValue> validator) {
+    String query =
+        "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM "
+            + bucketName
+            + " b UNNEST OBJECT_NAMES(b) AS v";
+    cluster.bucket(bucketName).waitUntilReady(Duration.ofSeconds(30));
+
+    QueryResult queryResult = query(query).join();
+    JsonObject jsonObject = queryResult.rowsAsObject().get(0);
+    Object columns = jsonObject.toMap().get("column");
+
+    Set<String> set = new HashSet<>();
+
+    if (columns != null) {
+      set.addAll((List<String>) columns);
+    }
+
+    validator.validate(set, entityClass);
+  }
+
   /**
    * Return the couchbase java sdk equivalent of a session - a collection - to the previously set
    * bucket

src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java

@@ -6,16 +6,22 @@
 package edu.ie3.datamodel.io.connectors;
 
 import edu.ie3.datamodel.exceptions.ConnectorException;
+import edu.ie3.datamodel.exceptions.FactoryException;
+import edu.ie3.datamodel.exceptions.SourceException;
 import edu.ie3.datamodel.io.IoUtil;
 import edu.ie3.datamodel.io.csv.*;
 import edu.ie3.datamodel.io.naming.FileNamingStrategy;
 import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation;
 import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
 import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
+import edu.ie3.datamodel.io.source.SourceValidator;
 import edu.ie3.datamodel.models.UniqueEntity;
 import edu.ie3.datamodel.models.timeseries.TimeSeries;
 import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
 import edu.ie3.datamodel.models.value.Value;
+import edu.ie3.datamodel.utils.ExceptionUtils;
+import edu.ie3.datamodel.utils.Try;
+import edu.ie3.util.StringUtils;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -51,6 +57,59 @@ public CsvFileConnector(Path baseDirectoryName, FileNamingStrategy fileNamingStr
     this.fileNamingStrategy = fileNamingStrategy;
   }
 
+  @SuppressWarnings("unchecked")
+  public void validate(Map<Class<?>, SourceValidator<?>> pairs, String csvSep) {
+    List<FactoryException> exceptions = new ArrayList<>();
+
+    pairs.forEach(
+        ((clz, validator) -> {
+          Class<? extends UniqueEntity> c = (Class<? extends UniqueEntity>) clz;
+          SourceValidator<UniqueEntity> v = (SourceValidator<UniqueEntity>) validator;
+
+          Try.of(() -> getHeadlineFields(c, csvSep), SourceException.class)
+              .getData()
+              .flatMap(
+                  headline ->
+                      Try.ofVoid(() -> v.validate(headline, c), FactoryException.class)
+                          .getException())
+              .ifPresent(exceptions::add);
+        }));
+
+    if (!exceptions.isEmpty()) {
+      throw new FactoryException(
+          "The following exception(s) occurred during validation: "
+              + ExceptionUtils.getMessages(exceptions));
+    }
+  }
+
+  /**
+   * Method for extracting the headline fields of a csv file.
+   *
+   * @param entityClass that should be buildable from the source data
+   * @param csvSep separator for headline
+   * @return a set of headline fields
+   * @throws SourceException if the file could not be read
+   */
+  public Set<String> getHeadlineFields(Class<? extends UniqueEntity> entityClass, String csvSep)
+      throws SourceException {
+    try (BufferedReader reader = initReader(entityClass)) {
+      // extracting headline fields
+      return Arrays.stream(reader.readLine().split(csvSep + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1))
+          .map(
+              maybeStartEndQuotedString ->
+                  StringUtils.unquoteStartEnd(maybeStartEndQuotedString.trim())
+                      .replaceAll("\"{2}", "\"")
+                      .trim())
+          .collect(Collectors.toSet());
+    } catch (ConnectorException | IOException e) {
+      throw new SourceException(
+          "Unable to find file for entity '"
+              + entityClass.getSimpleName()
+              + "': "
+              + e.getMessage());
+    }
+  }
+
   public synchronized BufferedCsvWriter getOrInitWriter(
       Class<? extends UniqueEntity> clz, String[] headerElements, String csvSep)
       throws ConnectorException {

src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java

@@ -5,6 +5,7 @@
 */
 package edu.ie3.datamodel.io.connectors;
 
+import edu.ie3.datamodel.io.source.SourceValidator;
 import java.util.*;
 import java.util.function.BinaryOperator;
 import java.util.stream.Collectors;
@@ -38,6 +39,7 @@ public class InfluxDbConnector implements DataConnector {
         return maps;
       };
 
+  private final String databaseName;
   private final String scenarioName;
   private final InfluxDB session;
 
@@ -77,6 +79,7 @@ public InfluxDbConnector(
    */
   public InfluxDbConnector(
       InfluxDB session, String scenarioName, String databaseName, boolean createDb) {
+    this.databaseName = databaseName;
     this.scenarioName = scenarioName;
     this.session = session;
 
@@ -107,6 +110,26 @@ public InfluxDbConnector(String url, String databaseName) {
     this(url, databaseName, NO_SCENARIO, true, InfluxDB.LogLevel.NONE, BatchOptions.DEFAULTS);
   }
 
+  /**
+   * This method should be used to validate a given {@link InfluxDB}.
+   *
+   * @param entityClass class of the entity
+   * @param validator for validation
+   */
+  public final void validateDb(Class<?> entityClass, SourceValidator validator) {
+    QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName));
+    Map<String, Set<Map<String, String>>> tagResults = parseQueryResult(tagKeys);
+
+    QueryResult fieldKeys = session.query(new Query("SHOW FIELD KEYS ON " + databaseName));
+    Map<String, Set<Map<String, String>>> fieldResults = parseQueryResult(fieldKeys);
+
+    Set<String> set = new HashSet<>();
+    tagResults.values().forEach(v -> v.stream().map(m -> m.get("tagKey")).forEach(set::add));
+    fieldResults.values().forEach(v -> v.stream().map(m -> m.get("fieldKey")).forEach(set::add));
+
+    validator.validate(set, entityClass);
+  }
+
   /**
    * Create the database of this connector if it doesn't exist yet
    *

src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java

@@ -5,6 +5,7 @@
 */
 package edu.ie3.datamodel.io.connectors;
 
+import edu.ie3.datamodel.io.source.SourceValidator;
 import edu.ie3.util.StringUtils;
 import edu.ie3.util.TimeUtil;
 import java.sql.*;
@@ -41,6 +42,28 @@ public SqlConnector(String jdbcUrl, String userName, String password) {
     connectionProps.put("password", password);
   }
 
+  /**
+   * This method should be used to validate a given sql table.
+   *
+   * @param tableName name of the table
+   * @param entityClass class of the entity
+   * @param validator for validation
+   * @throws SQLException – if the connection could not be established
+   */
+  public void validateDBTable(String tableName, Class<?> entityClass, SourceValidator validator)
+      throws SQLException {
+    ResultSet rs = getConnection().getMetaData().getColumns(null, null, tableName, null);
+
+    Set<String> columnNames = new HashSet<>();
+
+    while (rs.next()) {
+      String name = rs.getString("COLUMN_NAME");
+      columnNames.add(StringUtils.snakeCaseToCamelCase(name));
+    }
+
+    validator.validate(columnNames, entityClass);
+  }
+
   /**
    * Executes the given query. For update queries please use {@link
    * SqlConnector#executeUpdate(String)}.