diff --git a/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/AppConfig.java b/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/AppConfig.java index ceb01b433..58825959f 100644 --- a/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/AppConfig.java +++ b/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/AppConfig.java @@ -28,6 +28,7 @@ public final class AppConfig { public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER"); public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD"); public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB"); + public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB"); public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB"); public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS"); public static final String KAFKA_APPLICATION_ID = CONFIG.getString("KAFKA_APPLICATION_ID"); diff --git a/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/api/API.java b/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/api/API.java index 867225c39..cbdf22150 100644 --- a/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/api/API.java +++ b/JeMPI_Apps/JeMPI_API/src/main/java/org/jembi/jempi/api/API.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.Logger; import org.jembi.jempi.AppConfig; import org.jembi.jempi.libapi.BackEnd; - +import org.jembi.jempi.shared.config.Config; import java.util.UUID; public final class API { @@ -23,6 +23,13 @@ private API() { public static void main(final String[] args) { try { + // PostgresConfig postgresConfig = new PostgresConfig("localhost", 5432, "jempi", "postgres", "postgres"); + Config.create(AppConfig.POSTGRESQL_IP, + AppConfig.POSTGRESQL_PORT, + AppConfig.POSTGRESQL_CONFIGURATION_DB, + AppConfig.POSTGRESQL_USER, + AppConfig.POSTGRESQL_PASSWORD); + new API().run(); } catch (Exception e) { LOGGER.error(e.getLocalizedMessage(), e); @@ -40,6 +47,7 @@ public Behavior create() { AppConfig.POSTGRESQL_PASSWORD, AppConfig.POSTGRESQL_NOTIFICATIONS_DB, AppConfig.POSTGRESQL_AUDIT_DB, + AppConfig.POSTGRESQL_CONFIGURATION_DB, AppConfig.KAFKA_BOOTSTRAP_SERVERS, "CLIENT_ID_API-" + UUID.randomUUID(), AppConfig.SYSTEM_CONFIG_DIR, diff --git a/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/AppConfig.java b/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/AppConfig.java index 496442ba1..72701886a 100644 --- a/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/AppConfig.java +++ b/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/AppConfig.java @@ -29,6 +29,7 @@ public final class AppConfig { public static final String POSTGRESQL_USERS_DB = CONFIG.getString("POSTGRESQL_USERS_DB"); public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB"); public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB"); + public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB"); public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS"); public static final String KAFKA_APPLICATION_ID = CONFIG.getString("KAFKA_APPLICATION_ID"); private static final String[] DGRAPH_ALPHA_HOSTS = CONFIG.getString("DGRAPH_HOSTS").split(","); diff --git a/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/api/APIKC.java b/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/api/APIKC.java index 2f6af9ff8..968e06ba5 100644 --- a/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/api/APIKC.java +++ b/JeMPI_Apps/JeMPI_API_KC/src/main/java/org/jembi/jempi/api/APIKC.java @@ -40,6 +40,7 @@ public Behavior create() { AppConfig.POSTGRESQL_PASSWORD, AppConfig.POSTGRESQL_NOTIFICATIONS_DB, AppConfig.POSTGRESQL_AUDIT_DB, + AppConfig.POSTGRESQL_CONFIGURATION_DB, AppConfig.KAFKA_BOOTSTRAP_SERVERS, "CLIENT_ID_API_KC-" + UUID.randomUUID(), AppConfig.SYSTEM_CONFIG_DIR, diff --git a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/AppConfig.java b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/AppConfig.java index 906b9b586..512710e54 100644 --- a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/AppConfig.java +++ b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/AppConfig.java @@ -28,6 +28,7 @@ public final class AppConfig { public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER"); public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD"); public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB"); + public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB"); public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB"); public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS"); private static final String[] DGRAPH_ALPHA_HOSTS = CONFIG.getString("DGRAPH_HOSTS").split(","); diff --git a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/BackEnd.java b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/BackEnd.java index 92eb144ce..b2b7a95fe 100644 --- a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/BackEnd.java +++ b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/BackEnd.java @@ -24,6 +24,7 @@ public final class BackEnd extends AbstractBehavior { private final String pgPassword; private final String pgNotificationsDb; private final String pgAuditDb; + private final String pgConfigurationDb; private final PsqlNotifications psqlNotifications; private LibMPI libMPI = null; private String[] dgraphHosts = null; @@ -40,6 +41,7 @@ private BackEnd( final String sqlPassword, final String sqlNotificationsDb, final String sqlAuditDb, + final String sqlConfigurationDb, final String kafkaBootstrapServers, final String kafkaClientId) { super(context); @@ -53,6 +55,7 @@ private BackEnd( this.pgPassword = sqlPassword; this.pgNotificationsDb = sqlNotificationsDb; this.pgAuditDb = sqlAuditDb; + this.pgConfigurationDb = sqlConfigurationDb; psqlNotifications = new PsqlNotifications(sqlIP, sqlPort, sqlNotificationsDb, sqlUser, sqlPassword); openMPI(kafkaBootstrapServers, kafkaClientId, debugLevel); } catch (Exception e) { @@ -72,6 +75,7 @@ public static Behavior create( final String sqlPassword, final String sqlNotificationsDb, final String sqlAuditDb, + final String sqlConfigurationDb, final String kafkaBootstrapServers, final String kafkaClientId) { return Behaviors.setup(context -> new BackEnd(level, @@ -84,6 +88,7 @@ public static Behavior create( sqlPassword, sqlNotificationsDb, sqlAuditDb, + sqlConfigurationDb, kafkaBootstrapServers, kafkaClientId)); } diff --git a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/api/BackupRestoreAPI.java b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/api/BackupRestoreAPI.java index b7a818728..c7afb7101 100644 --- a/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/api/BackupRestoreAPI.java +++ b/JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/api/BackupRestoreAPI.java @@ -41,6 +41,7 @@ public Behavior create() { AppConfig.POSTGRESQL_PASSWORD, AppConfig.POSTGRESQL_NOTIFICATIONS_DB, AppConfig.POSTGRESQL_AUDIT_DB, + AppConfig.POSTGRESQL_CONFIGURATION_DB, AppConfig.KAFKA_BOOTSTRAP_SERVERS, "CLIENT_ID_API-" + UUID.randomUUID()), "BackEnd"); context.watch(backEnd); diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample b/JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample index 1efeeef99..cbe4a5b7e 100644 --- a/JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample +++ b/JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample @@ -4,6 +4,7 @@ POSTGRESQL_USER=postgres POSTGRESQL_PASSWORD= POSTGRESQL_USERS_DB= POSTGRESQL_NOTIFICATIONS_DB= +POSTGRESQL_CONFIGURATION_DB= POSTGRESQL_AUDIT_DB= POSTGRESQL_KC_TEST_DB= KAFKA_BOOTSTRAP_SERVERS=127.0.0.1 diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperConfig.java b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperConfig.java index 421c45897..9962cdd01 100644 --- a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperConfig.java +++ b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperConfig.java @@ -16,12 +16,16 @@ public class BootstrapperConfig { public final String POSTGRESQL_DATABASE; public final String POSTGRESQL_USERS_DB; public final String POSTGRESQL_NOTIFICATIONS_DB; + public final String POSTGRESQL_CONFIGURATION_DB; public final String POSTGRESQL_AUDIT_DB; public final String POSTGRESQL_KC_TEST_DB; public final String KAFKA_BOOTSTRAP_SERVERS; public final String KAFKA_APPLICATION_ID; public final String[] DGRAPH_ALPHA_HOSTS; public final int[] DGRAPH_ALPHA_PORTS; + public final String API_CONFIG_REFERENCE_FILENAME; + public final String API_FIELDS_CONFIG_FILENAME; + public final String SYSTEM_CONFIG_DIR; public BootstrapperConfig(final Config parsedConfig) { POSTGRESQL_IP = parsedConfig.getString("POSTGRESQL_IP"); @@ -33,11 +37,15 @@ public BootstrapperConfig(final Config parsedConfig) { POSTGRESQL_USERS_DB = parsedConfig.getString("POSTGRESQL_USERS_DB"); POSTGRESQL_NOTIFICATIONS_DB = parsedConfig.getString("POSTGRESQL_NOTIFICATIONS_DB"); POSTGRESQL_AUDIT_DB = parsedConfig.getString("POSTGRESQL_AUDIT_DB"); + POSTGRESQL_CONFIGURATION_DB = parsedConfig.getString("POSTGRESQL_CONFIGURATION_DB"); POSTGRESQL_KC_TEST_DB = parsedConfig.getString("POSTGRESQL_KC_TEST_DB"); KAFKA_BOOTSTRAP_SERVERS = parsedConfig.getString("KAFKA_BOOTSTRAP_SERVERS"); KAFKA_APPLICATION_ID = parsedConfig.getString("KAFKA_APPLICATION_ID"); DGRAPH_ALPHA_HOSTS = parsedConfig.getString("DGRAPH_HOSTS").split(","); + API_CONFIG_REFERENCE_FILENAME = parsedConfig.getString("API_CONFIG_REFERENCE_FILENAME"); + API_FIELDS_CONFIG_FILENAME = parsedConfig.getString("API_FIELDS_CONFIG_FILENAME"); + SYSTEM_CONFIG_DIR = parsedConfig.getString("SYSTEM_CONFIG_DIR"); DGRAPH_ALPHA_PORTS = Arrays.stream(parsedConfig.getString("DGRAPH_PORTS").split(",")).mapToInt(s -> { try { return Integer.parseInt(s); diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/sql/postgres/PostgresDataBootstrapper.java b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/sql/postgres/PostgresDataBootstrapper.java index a0ed29ab6..4d1ac9d0f 100644 --- a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/sql/postgres/PostgresDataBootstrapper.java +++ b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/sql/postgres/PostgresDataBootstrapper.java @@ -2,14 +2,19 @@ import org.jembi.jempi.bootstrapper.data.DataBootstrapper; import org.jembi.jempi.bootstrapper.data.utils.DataBootstraperConsts; - +import org.jembi.jempi.shared.models.GlobalConstants; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.sql.SQLException; +import java.sql.*; import java.util.List; import java.util.stream.Collectors; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; public class PostgresDataBootstrapper extends DataBootstrapper { @@ -37,6 +42,7 @@ protected List getAllDbSchemas() { new DBSchemaDetails(this.loadedConfig.POSTGRESQL_USERS_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_USERS_DB), new DBSchemaDetails(this.loadedConfig.POSTGRESQL_NOTIFICATIONS_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_NOTIFICATION_DB), new DBSchemaDetails(this.loadedConfig.POSTGRESQL_AUDIT_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_AUDIT_DB), + new DBSchemaDetails(this.loadedConfig.POSTGRESQL_CONFIGURATION_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_CONFIGURATION_DB), new DBSchemaDetails(this.loadedConfig.POSTGRESQL_KC_TEST_DB, null) ); } @@ -112,8 +118,57 @@ public Boolean deleteData() throws SQLException { } @Override - public Boolean resetAll() throws SQLException { + public Boolean resetAll() throws SQLException, IOException { LOGGER.info("Resetting Postgres data and schemas."); - return this.deleteData() && this.deleteTables() && this.createSchema(); + return this.deleteData() && this.deleteTables() && this.createSchema() && this.insertConfigurationData(); + } + + public Boolean insertConfigurationData() throws SQLException, IOException { + String configurationTable = "configuration"; + String[][] configFiles = getConfigFiles(configurationTable); + String insertQuery = String.format("INSERT INTO %s (key, json) VALUES (?, ?::jsonb)", configurationTable); + postgresDALLib.runQuery(connection -> { + PreparedStatement pstmt = connection.prepareStatement(insertQuery); + for (String[] configFile : configFiles) { + String filePath = configFile[0]; + String key = configFile[1]; + try { + String jsonContent = readJsonFile(filePath); + pstmt.setString(1, key); + pstmt.setString(2, jsonContent); + pstmt.addBatch(); + LOGGER.info("Added " + key + " configuration to batch"); + } catch (IOException e) { + // Handle the exception, e.g., log it or throw a runtime exception + throw new RuntimeException("Error reading JSON file", e); + } + } + pstmt.executeBatch(); + LOGGER.info("All configuration data inserted successfully"); + return null; + }, true, this.loadedConfig.POSTGRESQL_CONFIGURATION_DB); + return true; + } + + private String[][] getConfigFiles(final String configurationTable) { + String configDir = Optional.of(Paths.get(this.loadedConfig.SYSTEM_CONFIG_DIR)) + .filter(Files::exists) + .map(Path::toString) + .orElseGet(() -> System.getenv("SYSTEM_CSV_DIR")); + + LOGGER.info("Inserting configuration data into " + configurationTable); + LOGGER.info("configDir " + configDir); + // Define an array of config file paths and their corresponding keys + String[][] configFiles = { + {configDir + "/" + this.loadedConfig.API_CONFIG_REFERENCE_FILENAME, GlobalConstants.CONFIGURATION_CONFIG_KEY}, + {configDir + "/" + this.loadedConfig.API_FIELDS_CONFIG_FILENAME, GlobalConstants.CONFIGURATION_CONFIG_API_KEY}, + // Add more configuration files as needed + }; + return configFiles; + } + + private String readJsonFile(final String filePath) throws IOException { + Path path = Paths.get(filePath); + return Files.readString(path); } } diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/utils/DataBootstraperConsts.java b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/utils/DataBootstraperConsts.java index 9df49d282..b80721c0f 100644 --- a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/utils/DataBootstraperConsts.java +++ b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/data/utils/DataBootstraperConsts.java @@ -6,4 +6,5 @@ protected DataBootstraperConsts() { } public static final String POSTGRES_INIT_SCHEMA_AUDIT_DB = "/data/postgres/audit-schema.sql"; public static final String POSTGRES_INIT_SCHEMA_NOTIFICATION_DB = "/data/postgres/notifications-schema.sql"; public static final String POSTGRES_INIT_SCHEMA_USERS_DB = "/data/postgres/users-schema.sql"; + public static final String POSTGRES_INIT_SCHEMA_CONFIGURATION_DB = "/data/postgres/configuration-schema.sql"; } diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/configuration-schema.sql b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/configuration-schema.sql new file mode 100644 index 000000000..a72ec57b8 --- /dev/null +++ b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/configuration-schema.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS configuration ( + id SERIAL PRIMARY KEY, + key VARCHAR(255) NOT NULL UNIQUE, + json JSON NOT NULL, + dateCreated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + dateUpdated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create an index on the key column for faster lookups +CREATE INDEX IF NOT EXISTS idx_configuration_key ON configuration(key); + +-- INSERT INTO configuration (key, json) VALUES +-- ('config', pg_read_file('/app/conf_system/config.json')::json); \ No newline at end of file diff --git a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/AppConfig.java b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/AppConfig.java index b11c5d727..092f72b05 100644 --- a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/AppConfig.java +++ b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/AppConfig.java @@ -33,6 +33,7 @@ public final class AppConfig { public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD"); public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB"); public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB"); + public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB"); public static final Integer CONTROLLER_HTTP_PORT = CONFIG.getInt("CONTROLLER_HTTP_PORT"); public static final String LINKER_IP = CONFIG.getString("LINKER_IP"); public static final Integer LINKER_HTTP_PORT = CONFIG.getInt("LINKER_HTTP_PORT"); diff --git a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/Main.java b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/Main.java index ce985ac78..bce0df7d7 100644 --- a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/Main.java +++ b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/Main.java @@ -93,6 +93,7 @@ private void run() { LOGGER.info("CONFIG: {} {} {} {} {}", AppConfig.POSTGRESQL_NOTIFICATIONS_DB, AppConfig.POSTGRESQL_AUDIT_DB, + AppConfig.POSTGRESQL_CONFIGURATION_DB, AppConfig.KAFKA_BOOTSTRAP_SERVERS, AppConfig.KAFKA_APPLICATION_ID, AppConfig.KAFKA_CLIENT_ID); diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java index eadc5db2b..283d6a736 100644 --- a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java @@ -4,7 +4,6 @@ import akka.actor.typed.Behavior; import akka.actor.typed.javadsl.*; import akka.http.javadsl.server.directives.FileInfo; -import com.fasterxml.jackson.databind.ObjectMapper; import io.vavr.control.Either; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -15,6 +14,7 @@ import org.jembi.jempi.libmpi.LibMPI; import org.jembi.jempi.libmpi.MpiGeneralError; import org.jembi.jempi.libmpi.MpiServiceError; +import org.jembi.jempi.shared.config.Config; import org.jembi.jempi.shared.models.*; import org.jembi.jempi.shared.models.ConfigurationModel.Configuration; import org.jembi.jempi.shared.models.dashboard.NotificationStats; @@ -24,23 +24,26 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.*; + public final class BackEnd extends AbstractBehavior { private static final Logger LOGGER = LogManager.getLogger(BackEnd.class); + private final PsqlClient psqlClient; private final String pgIP; + private final int pgPort; private final String pgUser; private final String pgPassword; private final String pgNotificationsDb; private final String pgAuditDb; private final PsqlNotifications psqlNotifications; private final PsqlAuditTrail psqlAuditTrail; + private final String pgConfigurationDb; private final String systemConfigDirectory; private final String configReferenceFileName; private final String configMasterFileName; @@ -49,6 +52,8 @@ public final class BackEnd extends AbstractBehavior { private String[] dgraphHosts = null; private int[] dgraphPorts = null; + private final PostgresClientDao postgresClientDao; + private BackEnd( final Level debugLevel, final ActorContext context, @@ -60,6 +65,7 @@ private BackEnd( final String sqlPassword, final String sqlNotificationsDb, final String sqlAuditDb, + final String sqlConfigurationDb, final String kafkaBootstrapServers, final String kafkaClientId, final String systemConfigDirectory, @@ -72,11 +78,12 @@ private BackEnd( this.dgraphHosts = dgraphHosts; this.dgraphPorts = dgraphPorts; this.pgIP = sqlIP; - Integer pgPort = sqlPort; + this.pgPort = sqlPort; this.pgUser = sqlUser; this.pgPassword = sqlPassword; this.pgNotificationsDb = sqlNotificationsDb; this.pgAuditDb = sqlAuditDb; + this.pgConfigurationDb = sqlConfigurationDb; this.systemConfigDirectory = systemConfigDirectory; this.configReferenceFileName = configReferenceFileName; this.configMasterFileName = configMasterFileName; @@ -84,6 +91,9 @@ private BackEnd( psqlNotifications = new PsqlNotifications(sqlIP, sqlPort, sqlNotificationsDb, sqlUser, sqlPassword); psqlAuditTrail = new PsqlAuditTrail(sqlIP, sqlPort, sqlAuditDb, sqlUser, sqlPassword); openMPI(kafkaBootstrapServers, kafkaClientId, debugLevel); + psqlClient = new PsqlClient(sqlIP, sqlPort, sqlConfigurationDb, sqlUser, sqlPassword); + Config.create(sqlIP, sqlPort, sqlConfigurationDb, sqlUser, sqlPassword); + this.postgresClientDao = PostgresClientDaoImpl.create(sqlIP, sqlPort, sqlConfigurationDb, sqlUser, sqlPassword); } catch (Exception e) { LOGGER.error(e.getMessage(), e); throw e; @@ -101,6 +111,7 @@ public static Behavior create( final String sqlPassword, final String sqlNotificationsDb, final String sqlAuditDb, + final String sqlConfigurationDb, final String kafkaBootstrapServers, final String kafkaClientId, final String systemConfigDirectory, @@ -117,6 +128,7 @@ public static Behavior create( sqlPassword, sqlNotificationsDb, sqlAuditDb, + sqlConfigurationDb, kafkaBootstrapServers, kafkaClientId, systemConfigDirectory, @@ -494,76 +506,32 @@ private Behavior getSqlDashboardDataHandler(final SQLDashboardDataRequest } private Behavior getConfigurationHandler(final GetConfigurationRequest request) { - Path configMasterJsonFilePath = Paths.get(systemConfigDirectory, configMasterFileName); - Path configReferenceJsonFilePath = Paths.get(systemConfigDirectory, configReferenceFileName); - - Path configFilePath = configMasterJsonFilePath; - if (!Files.exists(configFilePath)) { - configFilePath = configReferenceJsonFilePath; - } - try { - String configFileContent = new String(Files.readAllBytes(configFilePath), StandardCharsets.UTF_8); - ObjectMapper mapper = new ObjectMapper(); - Configuration configuration = mapper.readValue(configFileContent, Configuration.class); + Configuration configuration = postgresClientDao.getConfiguration(GlobalConstants.CONFIGURATION_CONFIG_KEY); request.replyTo.tell(new GetConfigurationResponse(configuration)); - } catch (Exception exception) { - LOGGER.error("getConfigurationHandler failed with error: {}", exception.getMessage()); + } catch (Exception e) { + LOGGER.error("getConfigurationHandler failed with error: {}", e.getMessage()); } - return Behaviors.same(); } private Behavior getFieldsConfigurationHandler(final GetFieldsConfigurationRequest request) { - final var separator = FileSystems.getDefault().getSeparator(); - final String configDir = System.getenv("SYSTEM_CONFIG_DIRS"); - Path filePath = Paths.get(""); // Start with an empty path - // Create ubuntuFilePath - Path ubuntuFilePath = new File(String.format("%sapp%sconf_system%s%s", separator, separator, separator, fieldsConfigurationFileName)).toPath(); - // Check if ubuntuFilePath exists - if (Files.exists(ubuntuFilePath)) { - filePath = ubuntuFilePath; - } else { - // If ubuntuFilePath does not exist, assign the alternative path - filePath = Paths.get(configDir, "config-api.json"); - } try { - String configFileContent = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8); - FieldsConfiguration fieldsConfiguration = AppUtils.OBJECT_MAPPER.readValue(configFileContent, FieldsConfiguration.class); - ArrayList fields = new ArrayList<>(); - fields.addAll(fieldsConfiguration.systemFields()); - fields.addAll(fieldsConfiguration.fields()); + List fields = postgresClientDao.getFieldsConfiguration(GlobalConstants.CONFIGURATION_CONFIG_API_KEY); request.replyTo.tell(new GetFieldsConfigurationResponse(fields)); - } catch (Exception exception) { - LOGGER.error("getFieldsConfigurationHandler failed with error: {}", exception.getLocalizedMessage()); + } catch (Exception e) { + LOGGER.error("getFieldsConfigurationHandler failed with error: {}", e.getMessage()); } - return Behaviors.same(); } private Behavior postConfigurationHandler(final PostConfigurationRequest request) { - Path configMasterJsonFilePath = Paths.get(systemConfigDirectory, configMasterFileName); - ObjectMapper objectMapper = new ObjectMapper(); - Configuration configJson = request.configuration; - - if (configJson == null) { - request.replyTo.tell(new PostConfigurationResponse("error: configuration is missing")); - LOGGER.error("postConfigurationHandler failed: configuration is missing in the request"); - return Behaviors.same(); - } - try { - String jsonConfig = objectMapper.writeValueAsString(configJson); - Files.write(configMasterJsonFilePath, - jsonConfig.getBytes(StandardCharsets.UTF_8), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING); + postgresClientDao.saveConfiguration(request.configuration, GlobalConstants.CONFIGURATION_CONFIG_KEY); request.replyTo.tell(new PostConfigurationResponse("ok")); - } catch (Exception exception) { - LOGGER.error("postConfigurationHandler failed with error: {}", exception.getMessage()); - request.replyTo.tell(new PostConfigurationResponse("error: " + exception.getMessage())); + } catch (Exception e) { + LOGGER.error("postConfigurationHandler failed with error: {}", e.getMessage()); } - return Behaviors.same(); } diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDao.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDao.java new file mode 100644 index 000000000..c78aa504b --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDao.java @@ -0,0 +1,13 @@ +package org.jembi.jempi.libapi; + +import org.jembi.jempi.shared.models.ConfigurationModel.Configuration; +import org.jembi.jempi.shared.models.FieldsConfiguration; +import java.util.List; + +public interface PostgresClientDao { + void connect(); + void disconnect(); + Configuration getConfiguration(String configKey); + List getFieldsConfiguration(String configKey); + void saveConfiguration(Configuration configuration, String configKey); +} diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDaoImpl.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDaoImpl.java new file mode 100644 index 000000000..3529e06e4 --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PostgresClientDaoImpl.java @@ -0,0 +1,178 @@ +package org.jembi.jempi.libapi; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jembi.jempi.shared.models.ConfigurationModel.Configuration; +import org.jembi.jempi.shared.models.FieldsConfiguration; +import org.jembi.jempi.shared.utils.AppUtils; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +/** + * Implementation of PostgresClientDao interface for interacting with a PostgreSQL database. + * This class can be extended to provide custom behavior for database operations. + */ +public final class PostgresClientDaoImpl implements PostgresClientDao { + private static final Logger LOGGER = LogManager.getLogger(PostgresClientDaoImpl.class); + private final PsqlClient psqlClient; + + /** + * Constructs a new PostgresClientDaoImpl with the given database connection parameters. + * + * @param pgIP The IP address of the PostgreSQL server + * @param pgPort The port number of the PostgreSQL server + * @param pgDatabase The name of the database to connect to + * @param pgUser The username for database authentication + * @param pgPassword The password for database authentication + */ + private PostgresClientDaoImpl( + final String pgIP, + final int pgPort, + final String pgDatabase, + final String pgUser, + final String pgPassword) { + this.psqlClient = new PsqlClient(pgIP, pgPort, pgDatabase, pgUser, pgPassword); + } + + /** + * Creates a new instance of PostgresClientDaoImpl with the given database connection parameters. + * + * @param ip The IP address of the PostgreSQL server + * @param port The port number of the PostgreSQL server + * @param db The name of the database to connect to + * @param user The username for database authentication + * @param password The password for database authentication + */ + public static PostgresClientDaoImpl create( + final String ip, + final int port, + final String db, + final String user, + final String password) { + return new PostgresClientDaoImpl( + ip, + port, + db, + user, + password); + } + + /** + * Establishes a connection to the PostgreSQL database. + * This method can be overridden to provide custom connection logic. + * + * @throws SQLException if a database access error occurs + */ + @Override + public void connect() { + LOGGER.info("Connecting to PostgreSQL database"); + psqlClient.connect(); + LOGGER.info("Successfully connected to PostgreSQL database"); + } + + /** + * Closes the connection to the PostgreSQL database. + * This method can be overridden to provide custom disconnection logic. + * + * @throws SQLException if a database access error occurs + */ + @Override + public void disconnect() { + LOGGER.info("Disconnecting from PostgreSQL database"); + psqlClient.disconnect(); + LOGGER.info("Successfully disconnected from PostgreSQL database"); + } + + /** + * Retrieves the current configuration from the database. + * This method can be overridden to provide custom configuration retrieval logic. + * + * @return The Configuration object, or null if no configuration is found + * @throws SQLException if a database access error occurs or the retrieved JSON is invalid + */ + @Override + public Configuration getConfiguration(final String configKey) { + this.connect(); + LOGGER.info("Retrieving configuration from database"); + String sql = String.format("SELECT json FROM CONFIGURATION WHERE key = '%s' ORDER BY id DESC LIMIT 1", configKey); + try (PreparedStatement preparedStatement = psqlClient.prepareStatement(sql); + ResultSet rs = preparedStatement.executeQuery()) { + + if (rs.next()) { + String configFileContent = rs.getString("json"); + ObjectMapper mapper = new ObjectMapper(); + Configuration config = mapper.readValue(configFileContent, Configuration.class); + LOGGER.info("Successfully retrieved configuration from database"); + return config; + } else { + LOGGER.info("No configuration found in the database"); + return null; + } + } catch (Exception e) { + LOGGER.error(e); + } + this.disconnect(); + return null; + } + + /** + * Retrieves the fields configuration from the database. + * This method can be overridden to provide custom fields configuration retrieval logic. + * + * @return A List of FieldsConfiguration.Field objects, or null if no configuration is found + * @throws SQLException if a database access error occurs or the retrieved JSON is invalid + */ + @Override + public List getFieldsConfiguration(final String configKey) { + this.connect(); + LOGGER.info("Retrieving fields configuration from database"); + String sql = String.format("SELECT json FROM CONFIGURATION WHERE key = '%s' ORDER BY id DESC LIMIT 1", configKey); + try (PreparedStatement preparedStatement = psqlClient.prepareStatement(sql); + ResultSet rs = preparedStatement.executeQuery()) { + if (rs.next()) { + String configFileContent = rs.getString("json"); + FieldsConfiguration fieldsConfiguration = AppUtils.OBJECT_MAPPER.readValue(configFileContent, FieldsConfiguration.class); + ArrayList fields = new ArrayList<>(); + if (fieldsConfiguration != null && fieldsConfiguration.systemFields() != null && fieldsConfiguration.fields() != null) { + fields.addAll(fieldsConfiguration.systemFields()); + fields.addAll(fieldsConfiguration.fields()); + } + LOGGER.info("Successfully retrieved fields configuration from database"); + return fields; + } else { + LOGGER.info("No fields configuration found in the database"); + return null; + } + } catch (Exception e) { + LOGGER.error(e); + } + this.disconnect(); + return null; + } + + /** + * Saves the given configuration to the database. + * This method can be overridden to provide custom configuration saving logic. + * + * @param configuration The Configuration object to be saved + * @throws SQLException if a database access error occurs or the configuration cannot be converted to JSON + */ + @Override + public void saveConfiguration(final Configuration configuration, final String configKey) { + LOGGER.info("Saving configuration to database"); + this.connect(); + String sql = "UPDATE CONFIGURATION SET json = ?::jsonb WHERE key = ?"; + try (PreparedStatement preparedStatement = psqlClient.prepareStatement(sql)) { + String jsonConfig = AppUtils.OBJECT_MAPPER.writeValueAsString(configuration); + preparedStatement.setString(1, jsonConfig); + preparedStatement.setString(2, configKey); + int rowsAffected = preparedStatement.executeUpdate(); + LOGGER.info("Successfully saved configuration to database. Rows affected: {}", rowsAffected); + } catch (Exception e) { + LOGGER.error(e); + } + this.disconnect(); + } +} diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/Config.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/Config.java index fa277ca64..0e4c039ab 100644 --- a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/Config.java +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/Config.java @@ -1,34 +1,33 @@ package org.jembi.jempi.shared.config; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jembi.jempi.shared.config.input.JsonConfig; +import org.jembi.jempi.shared.utils.AppUtils; -import java.io.File; -import java.nio.file.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import com.fasterxml.jackson.core.JsonProcessingException; public final class Config { - public static final String CONFIG_FILE = "config.json"; + private static final Logger LOGGER = LogManager.getLogger(Config.class); public static final JsonConfig JSON_CONFIG; public static final FieldsConfig FIELDS_CONFIG; public static final InputInterfaceConfig INPUT_INTERFACE_CONFIG; public static final ApiConfig API_CONFIG; public static final LinkerConfig LINKER_CONFIG; public static final DGraphConfig DGRAPH_CONFIG; - + private static String pgIp; + private static int pgPort; + private static String pgDb; + private static String pgUser; + private static String pgPassword; static { - final var separator = FileSystems.getDefault().getSeparator(); - final String configDir = System.getenv("SYSTEM_CONFIG_DIRS"); - Path filePath = Paths.get(""); // Start with an empty path - // Create ubuntuFilePath - Path ubuntuFilePath = new File(String.format("%sapp%sconf_system%s%s", separator, separator, separator, CONFIG_FILE)).toPath(); - // Check if ubuntuFilePath exists - if (Files.exists(ubuntuFilePath)) { - filePath = ubuntuFilePath; - } else { - // If ubuntuFilePath does not exist, assign the alternative path for windows - filePath = Paths.get(configDir, CONFIG_FILE); - } - JSON_CONFIG = JsonConfig.fromJson(String.valueOf(filePath)); + JSON_CONFIG = retrieveConfigFromDatabase(); FIELDS_CONFIG = new FieldsConfig(JSON_CONFIG); INPUT_INTERFACE_CONFIG = new InputInterfaceConfig(JSON_CONFIG); API_CONFIG = new ApiConfig(JSON_CONFIG); @@ -36,7 +35,53 @@ public final class Config { DGRAPH_CONFIG = new DGraphConfig(JSON_CONFIG); } - private Config() { + private Config( + final String ip, + final int port, + final String db, + final String user, + final String password) { + pgIp = ip; + pgPort = port; + pgDb = db; + pgUser = user; + pgPassword = password; + } + + public static Config create( + final String ip, + final int port, + final String db, + final String user, + final String password) { + return new Config(ip, port, db, user, password); } + private static JsonConfig retrieveConfigFromDatabase() { + String port = String.valueOf(pgPort); + LOGGER.info("............................................................."); + LOGGER.info("Retrieved configuration from database: {} {} {} {} {}", port, pgIp, pgDb, pgUser, pgPassword); + LOGGER.info("............................................................."); + // Database connection details (should be externalized in production) + String url = "jdbc:postgresql://" + pgIp + ":" + port + "/" + pgDb; + String jsonConfig = null; + JsonConfig jsonConfigs = null; + try (Connection conn = DriverManager.getConnection(url, pgUser, pgPassword); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT json FROM configuration WHERE key = 'config'")) { + + if (rs.next()) { + jsonConfig = rs.getString("json"); + jsonConfigs = AppUtils.OBJECT_MAPPER.readValue(jsonConfig, JsonConfig.class); + LOGGER.info("............................................................."); + LOGGER.info("Retrieved configuration from database: {}", jsonConfigs); + LOGGER.info("............................................................."); + } else { + throw new RuntimeException("Configuration not found in the database"); + } + } catch (SQLException | JsonProcessingException e) { + throw new RuntimeException("Error retrieving configuration from database", e); + } + return jsonConfigs; + } } diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/PostgresConfig.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/PostgresConfig.java new file mode 100644 index 000000000..efe16f92f --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/config/PostgresConfig.java @@ -0,0 +1,39 @@ +package org.jembi.jempi.shared.config; + +public class PostgresConfig { + private final String ip; + private final int port; + private final String db; + private final String user; + private final String password; + + public PostgresConfig(final String ip, final int port, final String db, final String user, final String password) { + this.ip = ip; + this.port = port; + this.db = db; + this.user = user; + this.password = password; + } + + // Existing getters... + + /** + * Generates the JDBC URL for connecting to the PostgreSQL database. + * + * @return A string representing the JDBC URL. + */ + public String getJdbcUrl() { + return String.format("jdbc:postgresql://%s:%d/%s", ip, port, db); + } + + // Add a static factory method to create PostgresConfig from environment variables + public static PostgresConfig fromEnv() { + String ip = System.getenv("POSTGRES_IP"); + int port = Integer.parseInt(System.getenv("POSTGRES_PORT")); + String db = System.getenv("POSTGRES_DB"); + String user = System.getenv("POSTGRES_USER"); + String password = System.getenv("POSTGRES_PASSWORD"); + + return new PostgresConfig(ip, port, db, user, password); + } +} diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java index e7c6983d6..e68aed962 100644 --- a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java @@ -14,7 +14,7 @@ public final class GlobalConstants { public static final String TOPIC_AUDIT_TRAIL = "JeMPI-audit-trail"; public static final String TOPIC_NOTIFICATIONS = "JeMPI-notifications"; public static final String PSQL_TABLE_AUDIT_TRAIL = "audit_trail"; - + public static final String PSQL_TABLE_CONFIGURATION = "configuration"; /* * * HTTP SEGMENTS @@ -71,6 +71,10 @@ public final class GlobalConstants { public static final int TIMEOUT_GENERAL_SECS = 60; public static final int TIMEOUT_TEA_TIME_SECS = 30; + //Configuration Key + public static final String CONFIGURATION_CONFIG_KEY = "config"; + public static final String CONFIGURATION_CONFIG_API_KEY = "config-api"; + private GlobalConstants() { } diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java index 6d30d9206..dd949ecbf 100644 --- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java +++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java @@ -28,6 +28,7 @@ public final class AppConfig { // public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER"); // public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD"); // public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB"); + public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB"); public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS"); public static final String KAFKA_APPLICATION_ID_INTERACTIONS = CONFIG.getString("KAFKA_APPLICATION_ID_INTERACTIONS"); public static final String KAFKA_APPLICATION_ID_MU = CONFIG.getString("KAFKA_APPLICATION_ID_MU"); diff --git a/devops/linux/docker/backup_restore/postgres-backup.sh b/devops/linux/docker/backup_restore/postgres-backup.sh index 6371b05e9..f2c0d737d 100755 --- a/devops/linux/docker/backup_restore/postgres-backup.sh +++ b/devops/linux/docker/backup_restore/postgres-backup.sh @@ -20,7 +20,7 @@ DB_HOST="${POSTGRES_HOST:-localhost}" BACKUP_PATH="${POSTGRES_BACKUP_DIRECTORY}/$BACKUP_DATE_TIME" OLD_LOGS_DIR="${BACKUP_PATH}/old_logs" # Directory to store old logs -databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB") +databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB") # Check and Create Backup Directory and Old Logs Directory [ ! -d "$BACKUP_PATH" ] && mkdir -p "$BACKUP_PATH" diff --git a/devops/linux/docker/backup_restore/postgres-restore.sh b/devops/linux/docker/backup_restore/postgres-restore.sh index 09701a6a5..f75aefa61 100755 --- a/devops/linux/docker/backup_restore/postgres-restore.sh +++ b/devops/linux/docker/backup_restore/postgres-restore.sh @@ -17,7 +17,7 @@ PGDATABASE="${POSTGRESQL_DATABASE}" BACKUP_DIR="${POSTGRES_BACKUP_DIRECTORY}/$BACKUP_FOLDER_NAME" -databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB") +databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB") for db in "${databases[@]}"; do diff --git a/devops/linux/docker/conf/env/conf-env-high-1-pc.template b/devops/linux/docker/conf/env/conf-env-high-1-pc.template index a2fbe2dbe..d40cfe5b4 100644 --- a/devops/linux/docker/conf/env/conf-env-high-1-pc.template +++ b/devops/linux/docker/conf/env/conf-env-high-1-pc.template @@ -118,6 +118,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB} export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB} export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB} export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB} +export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB} export KAFKA_SERVERS=${KAFKA_SERVERS} export DGRAPH_HOSTS=${DGRAPH_HOSTS} diff --git a/devops/linux/docker/conf/env/conf-env-low-1-pc.template b/devops/linux/docker/conf/env/conf-env-low-1-pc.template index f6fb7b416..944ad05d1 100644 --- a/devops/linux/docker/conf/env/conf-env-low-1-pc.template +++ b/devops/linux/docker/conf/env/conf-env-low-1-pc.template @@ -96,6 +96,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB} export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB} export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB} export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB} +export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB} export KAFKA_SERVERS=${KAFKA_SERVERS} export DGRAPH_HOSTS=${DGRAPH_HOSTS} diff --git a/devops/linux/docker/conf/env/create-env-linux-high-1.sh b/devops/linux/docker/conf/env/create-env-linux-high-1.sh index 804c12ca0..9f6794cd7 100755 --- a/devops/linux/docker/conf/env/create-env-linux-high-1.sh +++ b/devops/linux/docker/conf/env/create-env-linux-high-1.sh @@ -46,6 +46,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092,kafka-02:9092,kafka-03:9092" export DGRAPH_HOSTS="alpha-01,alpha-02,alpha-03" diff --git a/devops/linux/docker/conf/env/create-env-linux-low-1.sh b/devops/linux/docker/conf/env/create-env-linux-low-1.sh index bfce6efcc..09484bed5 100755 --- a/devops/linux/docker/conf/env/create-env-linux-low-1.sh +++ b/devops/linux/docker/conf/env/create-env-linux-low-1.sh @@ -32,6 +32,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092" export DGRAPH_HOSTS="alpha-01" diff --git a/devops/linux/docker/conf/env/create-env-wsl-low-1.sh b/devops/linux/docker/conf/env/create-env-wsl-low-1.sh index 467d49aa8..4a7be4712 100755 --- a/devops/linux/docker/conf/env/create-env-wsl-low-1.sh +++ b/devops/linux/docker/conf/env/create-env-wsl-low-1.sh @@ -27,6 +27,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092" export DGRAPH_HOSTS="alpha-01" diff --git a/devops/linux/docker/conf/postgres/config-configuration.sql b/devops/linux/docker/conf/postgres/config-configuration.sql new file mode 100644 index 000000000..3f7627f3b --- /dev/null +++ b/devops/linux/docker/conf/postgres/config-configuration.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS configuration ( + id SERIAL PRIMARY KEY, + key VARCHAR(255) NOT NULL UNIQUE, + json JSON NOT NULL, + dateCreated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + dateUpdated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create an index on the key column for faster lookups +CREATE INDEX IF NOT EXISTS idx_configuration_key ON configuration(key); + +\dt; diff --git a/devops/linux/docker/conf/stack/docker-stack-high-0.yml b/devops/linux/docker/conf/stack/docker-stack-high-0.yml index 8fe72873c..8ee793413 100644 --- a/devops/linux/docker/conf/stack/docker-stack-high-0.yml +++ b/devops/linux/docker/conf/stack/docker-stack-high-0.yml @@ -28,6 +28,10 @@ services: - type: bind source: ${DATA_POSTGRESQL_DB_DIR} target: /bitnami/postgresql + - type: bind + source: $DATA_SYSTEM_CONFIG_DIR + target: /app/conf_system + read_only: true environment: POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} @@ -517,6 +521,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} @@ -602,6 +607,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID_INTERACTIONS: app-id-lnk1 KAFKA_APPLICATION_ID_MU: app-id-lnk2 @@ -653,6 +659,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -719,6 +726,7 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -822,11 +830,15 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} POSTGRESQL_KC_TEST_DB: ${POSTGRESQL_KC_TEST_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-bootstrapper DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + SYSTEM_CONFIG_DIR: /app/conf_system deploy: mode: global @@ -840,6 +852,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-backuprestoreapi DGRAPH_HOSTS: ${DGRAPH_HOSTS} diff --git a/devops/linux/docker/conf/stack/docker-stack-high-1.yml b/devops/linux/docker/conf/stack/docker-stack-high-1.yml index 6ab2d9688..e261f1bdd 100644 --- a/devops/linux/docker/conf/stack/docker-stack-high-1.yml +++ b/devops/linux/docker/conf/stack/docker-stack-high-1.yml @@ -28,6 +28,10 @@ services: - type: bind source: ${DATA_POSTGRESQL_DB_DIR} target: /bitnami/postgresql + - type: bind + source: $DATA_SYSTEM_CONFIG_DIR + target: /app/conf_system + read_only: true environment: POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} @@ -517,6 +521,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} @@ -602,6 +607,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID_INTERACTIONS: app-id-lnk1 KAFKA_APPLICATION_ID_MU: app-id-lnk2 @@ -653,6 +659,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -719,6 +726,7 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -822,11 +830,15 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} POSTGRESQL_KC_TEST_DB: ${POSTGRESQL_KC_TEST_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-bootstrapper DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + SYSTEM_CONFIG_DIR: /app/conf_system deploy: mode: global @@ -840,6 +852,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-backuprestoreapi DGRAPH_HOSTS: ${DGRAPH_HOSTS} diff --git a/devops/linux/docker/conf/stack/docker-stack-low-0.yml b/devops/linux/docker/conf/stack/docker-stack-low-0.yml index f8c4d1daf..a9096fc2b 100644 --- a/devops/linux/docker/conf/stack/docker-stack-low-0.yml +++ b/devops/linux/docker/conf/stack/docker-stack-low-0.yml @@ -28,6 +28,10 @@ services: - type: bind source: ${DATA_POSTGRESQL_DB_DIR} target: /bitnami/postgresql + - type: bind + source: $DATA_SYSTEM_CONFIG_DIR + target: /app/conf_system + read_only: true environment: POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} @@ -306,6 +310,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} @@ -391,6 +396,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID_INTERACTIONS: app-id-lnk1 KAFKA_APPLICATION_ID_MU: app-id-lnk2 @@ -442,6 +448,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -508,6 +515,7 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -611,11 +619,15 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} POSTGRESQL_KC_TEST_DB: ${POSTGRESQL_KC_TEST_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-bootstrapper DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + SYSTEM_CONFIG_DIR: /app/conf_system deploy: mode: global @@ -629,6 +641,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-backuprestoreapi DGRAPH_HOSTS: ${DGRAPH_HOSTS} diff --git a/devops/linux/docker/conf/stack/docker-stack-low-1.yml b/devops/linux/docker/conf/stack/docker-stack-low-1.yml index f731c3997..de3a7c898 100644 --- a/devops/linux/docker/conf/stack/docker-stack-low-1.yml +++ b/devops/linux/docker/conf/stack/docker-stack-low-1.yml @@ -28,6 +28,10 @@ services: - type: bind source: ${DATA_POSTGRESQL_DB_DIR} target: /bitnami/postgresql + - type: bind + source: $DATA_SYSTEM_CONFIG_DIR + target: /app/conf_system + read_only: true environment: POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} @@ -306,6 +310,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} @@ -391,6 +396,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID_INTERACTIONS: app-id-lnk1 KAFKA_APPLICATION_ID_MU: app-id-lnk2 @@ -442,6 +448,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -508,6 +515,7 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-api DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -601,7 +609,7 @@ services: - type: bind source: $DATA_SYSTEM_CONFIG_DIR target: /app/conf_system - read_only: true + read_only: true environment: POSTGRESQL_IP: postgresql POSTGRESQL_PORT: 5432 @@ -611,11 +619,15 @@ services: POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} POSTGRESQL_KC_TEST_DB: ${POSTGRESQL_KC_TEST_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-bootstrapper DGRAPH_HOSTS: ${DGRAPH_HOSTS} DGRAPH_PORTS: ${DGRAPH_PORTS} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + SYSTEM_CONFIG_DIR: /app/conf_system deploy: mode: global @@ -629,6 +641,7 @@ services: POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} + POSTGRESQL_CONFIGURATION_DB: ${POSTGRESQL_CONFIGURATION_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_SERVERS} KAFKA_APPLICATION_ID: app-id-backuprestoreapi DGRAPH_HOSTS: ${DGRAPH_HOSTS} diff --git a/devops/linux/docker/helper/postgres/create-schema.sh b/devops/linux/docker/helper/postgres/create-schema.sh index 4ab0e3c2b..f767f1e04 100755 --- a/devops/linux/docker/helper/postgres/create-schema.sh +++ b/devops/linux/docker/helper/postgres/create-schema.sh @@ -31,9 +31,14 @@ pushd . docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgres) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_DATABASE} -c "CREATE DATABASE ${POSTGRESQL_USERS_DB}" fi + EXISTING_CONFIGURATION_DB=$(docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgres) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_DATABASE} -tAc "SELECT 1 FROM pg_database WHERE datname='${POSTGRESQL_CONFIGURATION_DB}'") + if [ -z "$EXISTING_CONFIGURATION_DB" ]; then + docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgres) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_DATABASE} -c "CREATE DATABASE ${POSTGRESQL_CONFIGURATION_DB}" + fi docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgresql) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_USERS_DB} -a -f /conf/config-users.sql docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgresql) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_NOTIFICATIONS_DB} -a -f /conf/config-notifications.sql docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgresql) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_AUDIT_DB} -a -f /conf/config-audit.sql + docker exec -e PGPASSWORD=${POSTGRESQL_PASSWORD} $(docker ps -q -f name=${STACK_NAME}_postgresql) psql -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_CONFIGURATION_DB} -a -f /conf/config-configuration.sql popd diff --git a/devops/windows/base-docker-linux/conf/env/conf-env-high-1-pc.template b/devops/windows/base-docker-linux/conf/env/conf-env-high-1-pc.template index f1ec5bf90..bfcb73fb7 100644 --- a/devops/windows/base-docker-linux/conf/env/conf-env-high-1-pc.template +++ b/devops/windows/base-docker-linux/conf/env/conf-env-high-1-pc.template @@ -123,6 +123,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB} export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB} export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB} export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB} +export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB} export KAFKA_SERVERS=${KAFKA_SERVERS} export DGRAPH_HOSTS=${DGRAPH_HOSTS} diff --git a/devops/windows/base-docker-linux/conf/env/create-env-linux-high-1.sh b/devops/windows/base-docker-linux/conf/env/create-env-linux-high-1.sh index d06f8011e..fe10ee2fd 100755 --- a/devops/windows/base-docker-linux/conf/env/create-env-linux-high-1.sh +++ b/devops/windows/base-docker-linux/conf/env/create-env-linux-high-1.sh @@ -47,6 +47,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092,kafka-02:9092,kafka-03:9092" export DGRAPH_HOSTS="alpha-01,alpha-02,alpha-03" diff --git a/devops/windows/base-docker-linux/conf/env/create-env-linux-low-1.sh b/devops/windows/base-docker-linux/conf/env/create-env-linux-low-1.sh index 70deaad4f..a02f13a2a 100755 --- a/devops/windows/base-docker-linux/conf/env/create-env-linux-low-1.sh +++ b/devops/windows/base-docker-linux/conf/env/create-env-linux-low-1.sh @@ -34,6 +34,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092" export DGRAPH_HOSTS="alpha-01" diff --git a/devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template b/devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template index a45cc3269..a84e57917 100644 --- a/devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template +++ b/devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template @@ -122,6 +122,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB} export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB} export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB} export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB} +export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB} export KAFKA_SERVERS=${KAFKA_SERVERS} export DGRAPH_HOSTS=${DGRAPH_HOSTS} diff --git a/devops/windows/base-docker-wsl/conf/env/create-env-linux-high-1.sh b/devops/windows/base-docker-wsl/conf/env/create-env-linux-high-1.sh index d06f8011e..fe10ee2fd 100644 --- a/devops/windows/base-docker-wsl/conf/env/create-env-linux-high-1.sh +++ b/devops/windows/base-docker-wsl/conf/env/create-env-linux-high-1.sh @@ -47,6 +47,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092,kafka-02:9092,kafka-03:9092" export DGRAPH_HOSTS="alpha-01,alpha-02,alpha-03" diff --git a/devops/windows/base-docker-wsl/conf/env/create-env-linux-low-1.sh b/devops/windows/base-docker-wsl/conf/env/create-env-linux-low-1.sh index 9c543cff8..da16f9a54 100644 --- a/devops/windows/base-docker-wsl/conf/env/create-env-linux-low-1.sh +++ b/devops/windows/base-docker-wsl/conf/env/create-env-linux-low-1.sh @@ -34,6 +34,7 @@ export POSTGRESQL_USERS_DB="users_db" export POSTGRESQL_NOTIFICATIONS_DB="notifications_db" export POSTGRESQL_AUDIT_DB="audit_db" export POSTGRESQL_KC_TEST_DB="kc_test_db" +export POSTGRESQL_CONFIGURATION_DB="configuration_db" export KAFKA_SERVERS="kafka-01:9092" export DGRAPH_HOSTS="alpha-01" diff --git a/devops/windows/deployment/backup_restore/postgres-backup.sh b/devops/windows/deployment/backup_restore/postgres-backup.sh index 3e39bec5c..c9a65574c 100644 --- a/devops/windows/deployment/backup_restore/postgres-backup.sh +++ b/devops/windows/deployment/backup_restore/postgres-backup.sh @@ -9,7 +9,7 @@ DB_HOST="${POSTGRES_HOST:-localhost}" BACKUP_PATH="${POSTGRES_BACKUP_DIRECTORY}/$(date +%Y%m%d_%H%M%S)" OLD_LOGS_DIR="${BACKUP_PATH}/old_logs" # Directory to store old logs -databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB") +databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB") # Check and Create Backup Directory and Old Logs Directory [ ! -d "$BACKUP_PATH" ] && mkdir -p "$BACKUP_PATH" diff --git a/devops/windows/deployment/backup_restore/postgres-restore.sh b/devops/windows/deployment/backup_restore/postgres-restore.sh index 082a9b68e..8967a7cf5 100644 --- a/devops/windows/deployment/backup_restore/postgres-restore.sh +++ b/devops/windows/deployment/backup_restore/postgres-restore.sh @@ -31,7 +31,7 @@ PGDATABASE="${POSTGRESQL_DATABASE}" BACKUP_DIR="${POSTGRES_BACKUP_DIRECTORY}/$BACKUP_FOLDER_NAME" -databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB") +databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB") for db in "${databases[@]}"; do