diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java index 6093e51f..b8fdbf5b 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java @@ -72,534 +72,562 @@ @Service @PropertySource("classpath:application.properties") public class UploadDataToServerImpl implements UploadDataToServer { - private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); - // rest URLs from server to consume local van data and to sync to DB server - @Value("${dataSyncUploadUrl}") - private String dataSyncUploadUrl; - - @Value("${BATCH_SIZE}") - private int BATCH_SIZE; - - @Autowired - private DataSyncRepository dataSyncRepository; - @Autowired - private DataSyncGroupsRepo dataSyncGroupsRepo; - @Autowired - private MasterVanRepo masterVanRepo; - - @Autowired - private SyncUtilityClassRepo syncutilityClassRepo; - @Autowired - private CookieUtil cookieUtil; -boolean criticalTableFailure = false; // Add this flag - - /** - * - * @param groupName - * @param Authorization - * @return - */ - public String getDataToSyncToServer(int vanID, String user, String Authorization, String token) throws Exception { - String syncData = null; - syncData = syncIntercepter(vanID, user, Authorization, token); - return syncData; - } - - /** - * - * @param Authorization - * @return - */ - public String syncIntercepter(int vanID, String user, String Authorization, String token) throws Exception { - // sync activity trigger - String serverAcknowledgement = startDataSync(vanID, user, Authorization, token); - return serverAcknowledgement; - } - - /** - * Enhanced startDataSync method with table-level and group-level tracking - * - * @param syncTableDetailsIDs - * @param Authorization - * @return - */ - - private String startDataSync(int vanID, String user, String Authorization, String token) throws Exception { - String serverAcknowledgement = null; - List> responseStatus = new ArrayList<>(); - ObjectMapper objectMapper = new ObjectMapper(); - - // fetch group masters - List dataSyncGroupList = dataSyncGroupsRepo.findByDeleted(false); - logger.debug("Fetched DataSyncGroups: {}", - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dataSyncGroupList)); - - for (DataSyncGroups dataSyncGroups : dataSyncGroupList) { - int groupId = dataSyncGroups.getSyncTableGroupID(); - String groupName = dataSyncGroups.getSyncTableGroupName(); - - List syncUtilityClassList = getVanAndServerColumns(groupId); - logger.debug("Fetched SyncUtilityClass for groupId {}: {}", groupId, - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncUtilityClassList)); - - // Track table-level results for this group - List> tableDetailsList = new ArrayList<>(); - boolean groupHasFailures = false; - boolean groupHasCriticalFailure = false; // Track critical failures at group level - - for (SyncUtilityClass obj : syncUtilityClassList) { - String tableKey = obj.getSchemaName() + "." + obj.getTableName(); - boolean tableHasError = false; - - // get data from DB to sync to server - List> syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), - obj.getVanColumnName()); - logger.debug("Fetched syncData for schema {} and table {}: {}", obj.getSchemaName(), obj.getTableName(), - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncData)); - - if (syncData != null && syncData.size() > 0) { - int dataSize = syncData.size(); - int startIndex = 0; - int fullBatchCount = dataSize / BATCH_SIZE; - int remainder = dataSize % BATCH_SIZE; - - // Track table-level success/failure counts - int totalRecords = dataSize; - int successfulRecords = 0; - int failedRecords = 0; - List tableFailureReasons = new ArrayList<>(); - - logger.info("Starting batch sync for schema: {}, table: {} with {} full batches and {} remainder", - obj.getSchemaName(), obj.getTableName(), fullBatchCount, remainder); - - // Process full batches - for (int i = 0; i < fullBatchCount && !tableHasError; i++) { - List> syncDataBatch = getBatchOfAskedSizeDataToSync(syncData, startIndex, - BATCH_SIZE); - - Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, - Authorization, token); - - if (syncResult == null) { - logger.error("Sync failed for batch {} in schema: {}, table: {}", i, obj.getSchemaName(), - obj.getTableName()); - tableHasError = true; - failedRecords += syncDataBatch.size(); - groupHasFailures = true; - tableFailureReasons.add("Batch " + i + " sync returned null response"); - break; - } - - String status = (String) syncResult.get("status"); - int batchSuccessCount = (Integer) syncResult.get("successCount"); - int batchFailCount = (Integer) syncResult.get("failCount"); - @SuppressWarnings("unchecked") - List batchFailureReasons = (List) syncResult.get("failureReasons"); - - successfulRecords += batchSuccessCount; - failedRecords += batchFailCount; - - if (batchFailureReasons != null && !batchFailureReasons.isEmpty()) { - tableFailureReasons.addAll(batchFailureReasons); - groupHasFailures = true; - } - - if (status.equals("Sync failed")) { - tableHasError = true; - break; - } + private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + // rest URLs from server to consume local van data and to sync to DB server + @Value("${dataSyncUploadUrl}") + private String dataSyncUploadUrl; + + @Value("${BATCH_SIZE}") + private int BATCH_SIZE; + + @Autowired + private DataSyncRepository dataSyncRepository; + @Autowired + private DataSyncGroupsRepo dataSyncGroupsRepo; + @Autowired + private MasterVanRepo masterVanRepo; + + @Autowired + private SyncUtilityClassRepo syncutilityClassRepo; + @Autowired + private CookieUtil cookieUtil; + boolean criticalTableFailure = false; // Add this flag + + /** + * + * @param groupName + * @param Authorization + * @return + */ + public String getDataToSyncToServer(int vanID, String user, String Authorization, String token) throws Exception { + String syncData = null; + syncData = syncIntercepter(vanID, user, Authorization, token); + return syncData; + } - startIndex += BATCH_SIZE; - } + /** + * + * @param Authorization + * @return + */ + public String syncIntercepter(int vanID, String user, String Authorization, String token) throws Exception { + // sync activity trigger + String serverAcknowledgement = startDataSync(vanID, user, Authorization, token); + return serverAcknowledgement; + } + + /** + * Enhanced startDataSync method with table-level and group-level tracking + * + * @param syncTableDetailsIDs + * @param Authorization + * @return + */ + + private String startDataSync(int vanID, String user, String Authorization, String token) throws Exception { + String serverAcknowledgement = null; + List> responseStatus = new ArrayList<>(); + ObjectMapper objectMapper = new ObjectMapper(); + + // fetch group masters + List dataSyncGroupList = dataSyncGroupsRepo.findByDeleted(false); + logger.debug("Fetched DataSyncGroups: {}", + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dataSyncGroupList)); + + for (DataSyncGroups dataSyncGroups : dataSyncGroupList) { + int groupId = dataSyncGroups.getSyncTableGroupID(); + String groupName = dataSyncGroups.getSyncTableGroupName(); + + List syncUtilityClassList = getVanAndServerColumns(groupId); + logger.debug("Fetched SyncUtilityClass for groupId {}: {}", groupId, + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncUtilityClassList)); + + // Track table-level results for this group + List> tableDetailsList = new ArrayList<>(); + boolean groupHasFailures = false; + boolean groupHasCriticalFailure = false; // Track critical failures at group level + + for (SyncUtilityClass obj : syncUtilityClassList) { + String tableKey = obj.getSchemaName() + "." + obj.getTableName(); + boolean tableHasError = false; + + // get data from DB to sync to server + List> syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), + obj.getVanColumnName()); + logger.debug("Fetched syncData for schema {} and table {}: {}", obj.getSchemaName(), obj.getTableName(), + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncData)); + + if (syncData != null && syncData.size() > 0) { + int dataSize = syncData.size(); + int startIndex = 0; + int fullBatchCount = dataSize / BATCH_SIZE; + int remainder = dataSize % BATCH_SIZE; + + // Track table-level success/failure counts + int totalRecords = dataSize; + int successfulRecords = 0; + int failedRecords = 0; + List tableFailureReasons = new ArrayList<>(); + + logger.info("Starting batch sync for schema: {}, table: {} with {} full batches and {} remainder", + obj.getSchemaName(), obj.getTableName(), fullBatchCount, remainder); + + // Process full batches + for (int i = 0; i < fullBatchCount && !tableHasError; i++) { + List> syncDataBatch = getBatchOfAskedSizeDataToSync(syncData, startIndex, + BATCH_SIZE); + + Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), + obj.getTableName(), + obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + Authorization, token); + + if (syncResult == null) { + logger.error("Sync failed for batch {} in schema: {}, table: {}", i, obj.getSchemaName(), + obj.getTableName()); + tableHasError = true; + failedRecords += syncDataBatch.size(); + groupHasFailures = true; + tableFailureReasons.add("Batch " + i + " sync returned null response"); + break; + } - // Process remainder batch if no error in full batches - if (!tableHasError && remainder > 0) { - List> syncDataBatch = getBatchOfAskedSizeDataToSync(syncData, startIndex, - remainder); - - Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, - Authorization, token); - - if (syncResult == null) { - logger.error("Sync failed for remaining data in schema: {}, table: {}", obj.getSchemaName(), - obj.getTableName()); - failedRecords += syncDataBatch.size(); - groupHasFailures = true; - tableFailureReasons.add("Remainder batch sync returned null response"); - } else { String status = (String) syncResult.get("status"); int batchSuccessCount = (Integer) syncResult.get("successCount"); int batchFailCount = (Integer) syncResult.get("failCount"); @SuppressWarnings("unchecked") List batchFailureReasons = (List) syncResult.get("failureReasons"); - + successfulRecords += batchSuccessCount; failedRecords += batchFailCount; - + if (batchFailureReasons != null && !batchFailureReasons.isEmpty()) { tableFailureReasons.addAll(batchFailureReasons); groupHasFailures = true; } - + if (status.equals("Sync failed")) { + tableHasError = true; + break; + } + + startIndex += BATCH_SIZE; + } + + // Process remainder batch if no error in full batches + if (!tableHasError && remainder > 0) { + List> syncDataBatch = getBatchOfAskedSizeDataToSync(syncData, startIndex, + remainder); + + Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), + obj.getTableName(), + obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + Authorization, token); + + if (syncResult == null) { + logger.error("Sync failed for remaining data in schema: {}, table: {}", obj.getSchemaName(), + obj.getTableName()); + failedRecords += syncDataBatch.size(); groupHasFailures = true; + tableFailureReasons.add("Remainder batch sync returned null response"); + } else { + String status = (String) syncResult.get("status"); + int batchSuccessCount = (Integer) syncResult.get("successCount"); + int batchFailCount = (Integer) syncResult.get("failCount"); + @SuppressWarnings("unchecked") + List batchFailureReasons = (List) syncResult.get("failureReasons"); + + successfulRecords += batchSuccessCount; + failedRecords += batchFailCount; + + if (batchFailureReasons != null && !batchFailureReasons.isEmpty()) { + tableFailureReasons.addAll(batchFailureReasons); + groupHasFailures = true; + } + + if (status.equals("Sync failed")) { + groupHasFailures = true; + } } } - } - // Determine table status - String tableStatus; - if (successfulRecords == totalRecords && failedRecords == 0) { - tableStatus = "success"; - } else if (failedRecords == totalRecords && successfulRecords == 0) { - tableStatus = "failed"; - groupHasCriticalFailure = true; // Mark as critical failure - groupHasFailures = true; - } else if (successfulRecords > 0 && failedRecords > 0) { - tableStatus = "partial"; - groupHasFailures = true; - } else { - tableStatus = "failed"; - groupHasCriticalFailure = true; - groupHasFailures = true; - } + // Determine table status + String tableStatus; + if (successfulRecords == totalRecords && failedRecords == 0) { + tableStatus = "success"; + } else if (failedRecords == totalRecords && successfulRecords == 0) { + tableStatus = "failed"; + // groupHasCriticalFailure = true; // Mark as critical failure + groupHasFailures = true; + } else if (successfulRecords > 0 && failedRecords > 0) { + tableStatus = "partial"; + groupHasFailures = true; + } else { + tableStatus = "failed"; + // groupHasCriticalFailure = true; + groupHasFailures = true; + } - // Create detailed table info with failure reasons - Map tableDetails = new HashMap<>(); - tableDetails.put("tableName", obj.getTableName()); - tableDetails.put("schemaName", obj.getSchemaName()); - tableDetails.put("status", tableStatus); - tableDetails.put("totalRecords", totalRecords); - tableDetails.put("successfulRecords", successfulRecords); - tableDetails.put("failedRecords", failedRecords); - - // Add failure reasons only if there are any failures - if (!tableFailureReasons.isEmpty()) { - tableDetails.put("failureReasons", tableFailureReasons); + // Create detailed table info with failure reasons + Map tableDetails = new HashMap<>(); + tableDetails.put("tableName", obj.getTableName()); + tableDetails.put("schemaName", obj.getSchemaName()); + tableDetails.put("status", tableStatus); + tableDetails.put("totalRecords", totalRecords); + tableDetails.put("successfulRecords", successfulRecords); + tableDetails.put("failedRecords", failedRecords); + + // Add failure reasons only if there are any failures + if (!tableFailureReasons.isEmpty()) { + tableDetails.put("failureReasons", tableFailureReasons); + } + + tableDetailsList.add(tableDetails); + + logger.info("Table sync summary - {}: {} (Success: {}, Failed: {}, Total: {}, Failure Reasons: {})", + tableKey, tableStatus, successfulRecords, failedRecords, totalRecords, + tableFailureReasons.isEmpty() ? "None" : tableFailureReasons); + + } else { + logger.info("No data to sync for schema {} and table {}", obj.getSchemaName(), obj.getTableName()); + + Map tableDetails = new HashMap<>(); + tableDetails.put("tableName", obj.getTableName()); + tableDetails.put("schemaName", obj.getSchemaName()); + tableDetails.put("status", "no_data"); + tableDetails.put("totalRecords", 0); + tableDetails.put("successfulRecords", 0); + tableDetails.put("failedRecords", 0); + tableDetailsList.add(tableDetails); } - - tableDetailsList.add(tableDetails); - logger.info("Table sync summary - {}: {} (Success: {}, Failed: {}, Total: {}, Failure Reasons: {})", - tableKey, tableStatus, successfulRecords, failedRecords, totalRecords, - tableFailureReasons.isEmpty() ? "None" : tableFailureReasons); + // Continue processing all tables in the group - NO BREAK HERE + } + // Determine overall group status based on table results + // String groupStatus; + // long successTables = tableDetailsList.stream() + // .filter(table -> "success".equals(table.get("status")) || + // "no_data".equals(table.get("status"))) + // .count(); + // long partialTables = tableDetailsList.stream() + // .filter(table -> "partial".equals(table.get("status"))) + // .count(); + // long failedTables = tableDetailsList.stream() + // .filter(table -> "failed".equals(table.get("status"))) + // .count(); + + // if (groupHasCriticalFailure) { + // // If group has critical failure (100% failed tables), mark as failed + // groupStatus = "failed"; + // } else if (failedTables == 0 && partialTables == 0) { + // // All tables succeeded or had no data + // groupStatus = "completed"; + // } else if (failedTables > 0 && successTables == 0 && partialTables == 0) { + // // All tables failed + // groupStatus = "failed"; + // } else { + // // Mixed results - some success, some partial, some failed + // groupStatus = "partial"; + // } + + // Determine overall group status based on table results + String groupStatus; + long successTables = tableDetailsList.stream() + .filter(table -> "success".equals(table.get("status")) || "no_data".equals(table.get("status"))) + .count(); + long partialTables = tableDetailsList.stream() + .filter(table -> "partial".equals(table.get("status"))) + .count(); + long failedTables = tableDetailsList.stream() + .filter(table -> "failed".equals(table.get("status"))) + .count(); + + if (failedTables == 0 && partialTables == 0) { + // All tables succeeded or had no data + groupStatus = "completed"; + } else if (failedTables == tableDetailsList.size()) { + // ALL tables failed (100% failure at group level) + groupStatus = "failed"; + } else if (successTables > 0 && (failedTables > 0 || partialTables > 0)) { + // Mixed results - some success with some failures/partials + groupStatus = "partial"; + } else if (partialTables > 0 && failedTables == 0) { + // Only partial and success tables (no complete failures) + groupStatus = "partial"; } else { - logger.info("No data to sync for schema {} and table {}", obj.getSchemaName(), obj.getTableName()); - - Map tableDetails = new HashMap<>(); - tableDetails.put("tableName", obj.getTableName()); - tableDetails.put("schemaName", obj.getSchemaName()); - tableDetails.put("status", "no_data"); - tableDetails.put("totalRecords", 0); - tableDetails.put("successfulRecords", 0); - tableDetails.put("failedRecords", 0); - tableDetailsList.add(tableDetails); + // All other mixed scenarios + groupStatus = "partial"; } - - // Continue processing all tables in the group - NO BREAK HERE + // Create group response + Map groupResponse = new HashMap<>(); + groupResponse.put("syncTableGroupID", groupId); + groupResponse.put("syncTableGroupName", groupName != null ? groupName : "Group " + groupId); + groupResponse.put("status", groupStatus); + groupResponse.put("tables", tableDetailsList); + groupResponse.put("summary", Map.of( + "totalTables", tableDetailsList.size(), + "successfulTables", successTables, + "partialTables", partialTables, + "failedTables", failedTables)); + + responseStatus.add(groupResponse); + + // Continue to next group - NO BREAK HERE, process ALL groups } - // Determine overall group status based on table results - String groupStatus; - long successTables = tableDetailsList.stream() - .filter(table -> "success".equals(table.get("status")) || "no_data".equals(table.get("status"))) - .count(); - long partialTables = tableDetailsList.stream() - .filter(table -> "partial".equals(table.get("status"))) - .count(); - long failedTables = tableDetailsList.stream() - .filter(table -> "failed".equals(table.get("status"))) - .count(); - - if (groupHasCriticalFailure) { - // If group has critical failure (100% failed tables), mark as failed - groupStatus = "failed"; - } else if (failedTables == 0 && partialTables == 0) { - // All tables succeeded or had no data - groupStatus = "completed"; - } else if (failedTables > 0 && successTables == 0 && partialTables == 0) { - // All tables failed - groupStatus = "failed"; + // Create final response + Map finalResponse = new HashMap<>(); + + // Check if any group failed completely + boolean anyGroupFailed = responseStatus.stream() + .anyMatch(group -> "failed".equals(((Map) group).get("status"))); + + // Check if there was any data to sync + boolean hasData = responseStatus.stream() + .anyMatch(group -> { + @SuppressWarnings("unchecked") + List> tables = (List>) ((Map) group) + .get("tables"); + return tables.stream().anyMatch(table -> !("no_data".equals(table.get("status")))); + }); + + if (!hasData) { + return "No data to sync"; + } else if (anyGroupFailed) { + finalResponse.put("response", "Data sync completed with failures"); + finalResponse.put("groupsProgress", responseStatus); + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(finalResponse); } else { - // Mixed results - some success, some partial, some failed - groupStatus = "partial"; + finalResponse.put("response", "Data sync completed successfully"); + finalResponse.put("groupsProgress", responseStatus); + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(finalResponse); } + } - // Create group response - Map groupResponse = new HashMap<>(); - groupResponse.put("syncTableGroupID", groupId); - groupResponse.put("syncTableGroupName", groupName != null ? groupName : "Group " + groupId); - groupResponse.put("status", groupStatus); - groupResponse.put("tables", tableDetailsList); - groupResponse.put("summary", Map.of( - "totalTables", tableDetailsList.size(), - "successfulTables", successTables, - "partialTables", partialTables, - "failedTables", failedTables)); - - responseStatus.add(groupResponse); - - // Continue to next group - NO BREAK HERE, process ALL groups + private List getVanAndServerColumns(Integer groupID) throws Exception { + List syncUtilityClassList = getVanAndServerColumnList(groupID); + logger.debug("Fetched SyncUtilityClass list for groupID {}: {}", groupID, syncUtilityClassList); + return syncUtilityClassList; } - // Create final response - Map finalResponse = new HashMap<>(); - - // Check if any group failed completely - boolean anyGroupFailed = responseStatus.stream() - .anyMatch(group -> "failed".equals(((Map) group).get("status"))); - - // Check if there was any data to sync - boolean hasData = responseStatus.stream() - .anyMatch(group -> { - @SuppressWarnings("unchecked") - List> tables = (List>) ((Map) group) - .get("tables"); - return tables.stream().anyMatch(table -> !("no_data".equals(table.get("status")))); - }); - - if (!hasData) { - return "No data to sync"; - } else if (anyGroupFailed) { - finalResponse.put("response", "Data sync completed with failures"); - finalResponse.put("groupsProgress", responseStatus); - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(finalResponse); - } else { - finalResponse.put("response", "Data sync completed successfully"); - finalResponse.put("groupsProgress", responseStatus); - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(finalResponse); + public List getVanAndServerColumnList(Integer groupID) throws Exception { + List syncUtilityClassList = syncutilityClassRepo + .findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(groupID, false); + logger.debug("Fetched SyncUtilityClass list from repository for groupID {}: {}", groupID, syncUtilityClassList); + return syncUtilityClassList; } -} - - - private List getVanAndServerColumns(Integer groupID) throws Exception { - List syncUtilityClassList = getVanAndServerColumnList(groupID); - logger.debug("Fetched SyncUtilityClass list for groupID {}: {}", groupID, syncUtilityClassList); - return syncUtilityClassList; - } - - public List getVanAndServerColumnList(Integer groupID) throws Exception { - List syncUtilityClassList = syncutilityClassRepo - .findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(groupID, false); - logger.debug("Fetched SyncUtilityClass list from repository for groupID {}: {}", groupID, syncUtilityClassList); - return syncUtilityClassList; - } - - private List> getDataToSync(String schemaName, String tableName, String columnNames) - throws Exception { - logger.info("Fetching data to sync for schema: {}, table: {}, columns: {}", schemaName, tableName, columnNames); - List> resultSetList = dataSyncRepository.getDataForGivenSchemaAndTable(schemaName, - tableName, columnNames); - if (resultSetList != null) { - logger.debug("Fetched {} records for schema '{}', table '{}'", resultSetList.size(), schemaName, tableName); - if (!resultSetList.isEmpty()) { - logger.debug("Sample record: {}", resultSetList.get(0)); - } - } else { - logger.debug("No records found for schema '{}', table '{}'", schemaName, tableName); - } - return resultSetList; - } - - private List> getBatchOfAskedSizeDataToSync(List> syncData, int startIndex, - int size) throws Exception { - List> syncDataOfBatchSize = syncData.subList(startIndex, (startIndex + size)); - return syncDataOfBatchSize; - } - -public Map syncDataToServer(int vanID, String schemaName, String tableName, - String vanAutoIncColumnName, String serverColumns, List> dataToBesync, - String user, String Authorization, String token) throws Exception { - - RestTemplate restTemplate = new RestTemplate(); - - // Configure timeouts - SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); - factory.setConnectTimeout(60000); // 60 seconds to connect - factory.setReadTimeout(600000); // 10 minutes to read response - restTemplate.setRequestFactory(factory); - - Integer facilityID = masterVanRepo.getFacilityID(vanID); - - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.serializeNulls(); - Gson gson = gsonBuilder.create(); - - Map dataMap = new HashMap<>(); - dataMap.put("schemaName", schemaName); - dataMap.put("tableName", tableName); - dataMap.put("vanAutoIncColumnName", vanAutoIncColumnName); - dataMap.put("serverColumns", serverColumns); - dataMap.put("syncData", dataToBesync); - dataMap.put("syncedBy", user); - if (facilityID != null) - dataMap.put("facilityID", facilityID); - - String requestOBJ = gson.toJson(dataMap); - HttpEntity request = RestTemplateUtil.createRequestEntity(requestOBJ, Authorization, "datasync"); - - logger.info("Syncing {} records for table {}", dataToBesync.size(), tableName); - - int successCount = 0; - int failCount = 0; - List successVanSerialNos = new ArrayList<>(); - List failedVanSerialNos = new ArrayList<>(); - List failureReasons = new ArrayList<>(); - - try { - ResponseEntity response = restTemplate.exchange(dataSyncUploadUrl, HttpMethod.POST, - request, String.class); - - logger.info("Response from server: status={}, hasBody={}, getBody={}", - response.getStatusCode(), response.hasBody(), response.getBody()); - - - if (response != null && response.hasBody()) { - try { - JSONObject obj = new JSONObject(response.getBody()); - - int statusCode = obj.optInt("statusCode", 200); - String errorMessage = obj.optString("errorMessage", ""); - - // Check for ACTUAL errors - status >= 400 OR errorMessage is not "Success" - if (statusCode >= 400 || (!errorMessage.isEmpty() && !errorMessage.equalsIgnoreCase("Success"))) { - logger.error("Server returned error: {}", errorMessage); - - // Mark all as failed - for (Map map : dataToBesync) { - failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add(errorMessage); + + private List> getDataToSync(String schemaName, String tableName, String columnNames) + throws Exception { + logger.info("Fetching data to sync for schema: {}, table: {}, columns: {}", schemaName, tableName, columnNames); + List> resultSetList = dataSyncRepository.getDataForGivenSchemaAndTable(schemaName, + tableName, columnNames); + if (resultSetList != null) { + logger.debug("Fetched {} records for schema '{}', table '{}'", resultSetList.size(), schemaName, tableName); + if (!resultSetList.isEmpty()) { + logger.debug("Sample record: {}", resultSetList.get(0)); } - failCount = failedVanSerialNos.size(); - - } else if (obj.has("data")) { - JSONObject dataObj = obj.getJSONObject("data"); - - if (dataObj.has("records")) { - JSONArray recordsArr = dataObj.getJSONArray("records"); - for (int i = 0; i < recordsArr.length(); i++) { - JSONObject record = recordsArr.getJSONObject(i); - String vanSerialNo = record.getString("vanSerialNo"); - boolean success = record.getBoolean("success"); - - if (success) { - successVanSerialNos.add(vanSerialNo); - successCount++; - } else { - failedVanSerialNos.add(vanSerialNo); - failCount++; - String reason = record.optString("reason", "Unknown error"); - failureReasons.add(reason); - } - } - } else if (tableName.equalsIgnoreCase("m_beneficiaryregidmapping")) { - String respMsg = dataObj.optString("response", ""); - - if (respMsg.toLowerCase().contains("success") && statusCode == 200) { - for (Map map : dataToBesync) { - successVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + } else { + logger.debug("No records found for schema '{}', table '{}'", schemaName, tableName); + } + return resultSetList; + } + + private List> getBatchOfAskedSizeDataToSync(List> syncData, int startIndex, + int size) throws Exception { + List> syncDataOfBatchSize = syncData.subList(startIndex, (startIndex + size)); + return syncDataOfBatchSize; + } + + public Map syncDataToServer(int vanID, String schemaName, String tableName, + String vanAutoIncColumnName, String serverColumns, List> dataToBesync, + String user, String Authorization, String token) throws Exception { + + RestTemplate restTemplate = new RestTemplate(); + + // Configure timeouts + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(60000); // 60 seconds to connect + factory.setReadTimeout(600000); // 10 minutes to read response + restTemplate.setRequestFactory(factory); + + Integer facilityID = masterVanRepo.getFacilityID(vanID); + + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.serializeNulls(); + Gson gson = gsonBuilder.create(); + + Map dataMap = new HashMap<>(); + dataMap.put("schemaName", schemaName); + dataMap.put("tableName", tableName); + dataMap.put("vanAutoIncColumnName", vanAutoIncColumnName); + dataMap.put("serverColumns", serverColumns); + dataMap.put("syncData", dataToBesync); + dataMap.put("syncedBy", user); + if (facilityID != null) + dataMap.put("facilityID", facilityID); + + String requestOBJ = gson.toJson(dataMap); + HttpEntity request = RestTemplateUtil.createRequestEntity(requestOBJ, Authorization, "datasync"); + + logger.info("Syncing {} records for table {}", dataToBesync.size(), tableName); + + int successCount = 0; + int failCount = 0; + List successVanSerialNos = new ArrayList<>(); + List failedVanSerialNos = new ArrayList<>(); + List failureReasons = new ArrayList<>(); + + try { + ResponseEntity response = restTemplate.exchange(dataSyncUploadUrl, HttpMethod.POST, + request, String.class); + + logger.info("Response from server: status={}, hasBody={}, getBody={}", + response.getStatusCode(), response.hasBody(), response.getBody()); + + if (response != null && response.hasBody()) { + try { + JSONObject obj = new JSONObject(response.getBody()); + + int statusCode = obj.optInt("statusCode", 200); + String errorMessage = obj.optString("errorMessage", ""); + + // Check for ACTUAL errors - status >= 400 OR errorMessage is not "Success" + if (statusCode >= 400 || (!errorMessage.isEmpty() && !errorMessage.equalsIgnoreCase("Success"))) { + logger.error("Server returned error: {}", errorMessage); + + // Mark all as failed + for (Map map : dataToBesync) { + failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + failureReasons.add(errorMessage); + } + failCount = failedVanSerialNos.size(); + + } else if (obj.has("data")) { + JSONObject dataObj = obj.getJSONObject("data"); + + if (dataObj.has("records")) { + JSONArray recordsArr = dataObj.getJSONArray("records"); + for (int i = 0; i < recordsArr.length(); i++) { + JSONObject record = recordsArr.getJSONObject(i); + String vanSerialNo = record.getString("vanSerialNo"); + boolean success = record.getBoolean("success"); + + if (success) { + successVanSerialNos.add(vanSerialNo); + successCount++; + } else { + failedVanSerialNos.add(vanSerialNo); + failCount++; + String reason = record.optString("reason", "Unknown error"); + failureReasons.add(reason); + } + } + } else if (tableName.equalsIgnoreCase("m_beneficiaryregidmapping")) { + String respMsg = dataObj.optString("response", ""); + + if (respMsg.toLowerCase().contains("success") && statusCode == 200) { + for (Map map : dataToBesync) { + successVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + } + successCount = successVanSerialNos.size(); + } else { + for (Map map : dataToBesync) { + failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + failureReasons.add(respMsg.isEmpty() ? "Sync failed" : respMsg); + } + failCount = failedVanSerialNos.size(); + } + } } - successCount = successVanSerialNos.size(); - } else { + } catch (JSONException e) { + logger.error("Failed to parse server response: {}", e.getMessage(), e); for (Map map : dataToBesync) { failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add(respMsg.isEmpty() ? "Sync failed" : respMsg); + failureReasons.add("Invalid server response"); } failCount = failedVanSerialNos.size(); } + } else { + logger.error("Empty response from server"); + for (Map map : dataToBesync) { + failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + failureReasons.add("Empty server response"); + } + failCount = failedVanSerialNos.size(); } - } - } catch (JSONException e) { - logger.error("Failed to parse server response: {}", e.getMessage(), e); - for (Map map : dataToBesync) { - failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add("Invalid server response"); - } - failCount = failedVanSerialNos.size(); - } -} - else { - logger.error("Empty response from server"); + + } catch (ResourceAccessException e) { + logger.error("Connection error during sync: {}", e.getMessage(), e); + + // Mark all as failed due to connection error for (Map map : dataToBesync) { failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add("Empty server response"); + failureReasons.add("Connection failed: " + e.getMessage()); + } + failCount = failedVanSerialNos.size(); + + } catch (Exception e) { + logger.error("Unexpected error during sync: {}", e.getMessage(), e); + + for (Map map : dataToBesync) { + failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); + failureReasons.add("Unexpected error: " + e.getMessage()); } failCount = failedVanSerialNos.size(); } - - } catch (ResourceAccessException e) { - logger.error("Connection error during sync: {}", e.getMessage(), e); - - // Mark all as failed due to connection error - for (Map map : dataToBesync) { - failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add("Connection failed: " + e.getMessage()); + + logger.info("Sync complete - Success: {}, Failed: {}", successCount, failCount); + + // Update processed flags + if (!successVanSerialNos.isEmpty()) { + dataSyncRepository.updateProcessedFlagInVan(schemaName, tableName, successVanSerialNos, + vanAutoIncColumnName, user, "P", "null"); } - failCount = failedVanSerialNos.size(); - - } catch (Exception e) { - logger.error("Unexpected error during sync: {}", e.getMessage(), e); - - for (Map map : dataToBesync) { - failedVanSerialNos.add(String.valueOf(map.get(vanAutoIncColumnName))); - failureReasons.add("Unexpected error: " + e.getMessage()); + if (!failedVanSerialNos.isEmpty()) { + String firstError = failureReasons.isEmpty() ? "Unknown error" : failureReasons.get(0); + dataSyncRepository.updateProcessedFlagInVan(schemaName, tableName, failedVanSerialNos, + vanAutoIncColumnName, user, "F", firstError); + } + + Map result = new HashMap<>(); + if (successCount > 0 && failCount == 0) { + result.put("status", "Data successfully synced"); + } else if (successCount > 0 && failCount > 0) { + result.put("status", "Partial success"); + } else { + result.put("status", "Sync failed"); } - failCount = failedVanSerialNos.size(); - } - logger.info("Sync complete - Success: {}, Failed: {}", successCount, failCount); - - // Update processed flags - if (!successVanSerialNos.isEmpty()) { - dataSyncRepository.updateProcessedFlagInVan(schemaName, tableName, successVanSerialNos, - vanAutoIncColumnName, user, "P", "null"); + result.put("successCount", successCount); + result.put("failCount", failCount); + + return result; } - if (!failedVanSerialNos.isEmpty()) { - String firstError = failureReasons.isEmpty() ? "Unknown error" : failureReasons.get(0); - dataSyncRepository.updateProcessedFlagInVan(schemaName, tableName, failedVanSerialNos, - vanAutoIncColumnName, user, "F", firstError); + + public StringBuilder getVanSerialNoListForSyncedData(String vanAutoIncColumnName, + List> dataToBesync) throws Exception { + // comma separated van serial no + StringBuilder vanSerialNos = new StringBuilder(); + + int pointer1 = 0; + for (Map map : dataToBesync) { + if (pointer1 == dataToBesync.size() - 1) + vanSerialNos.append(map.get(vanAutoIncColumnName.trim())); + else + vanSerialNos.append(map.get(vanAutoIncColumnName.trim()) + ","); + + pointer1++; + } + return vanSerialNos; } - Map result = new HashMap<>(); - if (successCount > 0 && failCount == 0) { - result.put("status", "Data successfully synced"); - } else if (successCount > 0 && failCount > 0) { - result.put("status", "Partial success"); - } else { - result.put("status", "Sync failed"); + public String getDataSyncGroupDetails() { + List dataSyncGroupList = dataSyncGroupsRepo.findByDeleted(false); + if (dataSyncGroupList != null) + return new Gson().toJson(dataSyncGroupList); + else + return null; } - - result.put("successCount", successCount); - result.put("failCount", failCount); - - return result; -} - - public StringBuilder getVanSerialNoListForSyncedData(String vanAutoIncColumnName, - List> dataToBesync) throws Exception { - // comma separated van serial no - StringBuilder vanSerialNos = new StringBuilder(); - - int pointer1 = 0; - for (Map map : dataToBesync) { - if (pointer1 == dataToBesync.size() - 1) - vanSerialNos.append(map.get(vanAutoIncColumnName.trim())); - else - vanSerialNos.append(map.get(vanAutoIncColumnName.trim()) + ","); - - pointer1++; - } - return vanSerialNos; - } - - public String getDataSyncGroupDetails() { - List dataSyncGroupList = dataSyncGroupsRepo.findByDeleted(false); - if (dataSyncGroupList != null) - return new Gson().toJson(dataSyncGroupList); - else - return null; - } } \ No newline at end of file