From 19da6db3835b89dde6d2c52e477f30edaa935fda Mon Sep 17 00:00:00 2001 From: pratik08sha Date: Fri, 24 Nov 2023 17:39:22 -0500 Subject: [PATCH 1/3] java version updated --- Dockerfile | 13 +++++++++++++ boot.sh | 14 ++++++++++++++ pom.xml | 18 +++++++++--------- .../odsmetadata/entity/BatchJobExecution.java | 2 +- .../entity/BatchJobExecutionParams.java | 2 +- .../odsmetadata/entity/BatchStepExecution.java | 2 +- .../odsmetadata/services/InfluxIOService.java | 2 +- src/main/resources/application.properties | 2 +- 8 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 Dockerfile create mode 100755 boot.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64cd1fa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM maven:3.9.5 AS build + +COPY src /home/app/src +COPY pom.xml /home/app +RUN mvn -f /home/app/pom.xml clean install -DskipTests + +FROM openjdk:21 +COPY --from=build /home/app/target/ods-metadata-0.0.1-SNAPSHOT.jar /usr/local/lib/ods-metadata-0.0.1-SNAPSHOT.jar +COPY boot.sh . + +EXPOSE 8083 +ENTRYPOINT ["bash","boot.sh"] +ENTRYPOINT ["java","-jar","/usr/local/lib/rods-metadata-0.0.1-SNAPSHOT.jar"] diff --git a/boot.sh b/boot.sh new file mode 100755 index 0000000..028d315 --- /dev/null +++ b/boot.sh @@ -0,0 +1,14 @@ +export EUREKA_URI=3.136.74.174:8090 +export EUREKA_PASS=W872INBJNWZcoDN +export EUREKA_USER=someUserName + +export COCKROACH_USER="monitoring" +export COCKROACH_PASS="9gLufC2BVxta3O6m" +export COCKROACH_URI="postgresql://onedatashare-627.j77.cockroachlabs.cloud:26257/onedatashare?sslmode=prefer" + +export INFLUX_TOKEN="CkCFcu0sca6udDVk3A0Bbj1rfwilGSEAp8A36Qy7LOhE2DhnVqAtoT1QpD9dKWwVIr4oIczft0ypWhRNNKN0XA==" +export INFLUX_ORG="10972f62ad2d59a6" + +mvn clean package -DskipTests + +java -Dspring.profiles.active=dev -jar target/ods-metadata-0.0.1-SNAPSHOT.jar diff --git a/pom.xml b/pom.xml index 6dfdeb8..e745547 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.6.2 + 3.1.0 org.onedatashare @@ -14,8 +14,8 @@ ods-metadata OneDataShare Service that exposes the Data in CDB to other services. - 11 - 2021.0.0 + 21 + 2022.0.3 @@ -64,7 +64,7 @@ org.springframework.cloud spring-cloud-starter-netflix-eureka-client - 3.1.0 + 4.0.2 org.postgresql @@ -79,7 +79,7 @@ org.projectlombok lombok - 1.18.24 + 1.18.30 org.projectlombok @@ -98,12 +98,12 @@ jakarta.validation jakarta.validation-api - 2.0.2 + 3.0.2 org.springframework spring-web - 5.3.16 + 6.0.9 @@ -133,7 +133,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.10.1 + 3.11.0 ${java.version} ${java.version} @@ -146,7 +146,7 @@ org.projectlombok lombok - 1.18.24 + 1.18.30 org.projectlombok diff --git a/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecution.java b/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecution.java index a6f197f..d6d4c6b 100644 --- a/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecution.java +++ b/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecution.java @@ -2,7 +2,7 @@ import lombok.Data; -import javax.persistence.*; +import jakarta.persistence.*; import java.sql.Timestamp; import java.util.List; diff --git a/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecutionParams.java b/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecutionParams.java index e89e114..c40e61b 100644 --- a/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecutionParams.java +++ b/src/main/java/org/onedatashare/odsmetadata/entity/BatchJobExecutionParams.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import javax.persistence.*; +import jakarta.persistence.*; import java.sql.Timestamp; @Data @Entity diff --git a/src/main/java/org/onedatashare/odsmetadata/entity/BatchStepExecution.java b/src/main/java/org/onedatashare/odsmetadata/entity/BatchStepExecution.java index e2fce13..5a89a4e 100644 --- a/src/main/java/org/onedatashare/odsmetadata/entity/BatchStepExecution.java +++ b/src/main/java/org/onedatashare/odsmetadata/entity/BatchStepExecution.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import javax.persistence.*; +import jakarta.persistence.*; import java.sql.Timestamp; @Data @Entity diff --git a/src/main/java/org/onedatashare/odsmetadata/services/InfluxIOService.java b/src/main/java/org/onedatashare/odsmetadata/services/InfluxIOService.java index 278464b..80a7c04 100644 --- a/src/main/java/org/onedatashare/odsmetadata/services/InfluxIOService.java +++ b/src/main/java/org/onedatashare/odsmetadata/services/InfluxIOService.java @@ -10,7 +10,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 27d97f9..8af2d43 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,7 +4,7 @@ server.port=8084 management.endpoints.web.exposure.include=* #Job Datasource config -spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:${COCKROACH_URI:postgresql://localhost:26257/job_details?sslmode=disable} From 5b38cb9cf6c1c312d214ae6496729de4e5993688 Mon Sep 17 00:00:00 2001 From: pratik08sha Date: Tue, 19 Dec 2023 18:23:42 -0500 Subject: [PATCH 2/3] service for batchJobDetails based on email and type --- .../controller/BatchJobController.java | 18 ++++++++++++++++++ .../repository/BatchJobRepository.java | 11 +++++++++++ .../odsmetadata/services/JobService.java | 9 +++++++++ 3 files changed, 38 insertions(+) diff --git a/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java b/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java index 267d185..9437ea6 100644 --- a/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java +++ b/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java @@ -1,5 +1,6 @@ package org.onedatashare.odsmetadata.controller; + import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.onedatashare.odsmetadata.model.BatchJobData; @@ -18,6 +19,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.onedatashare.odsmetadata.entity.BatchJobExecution; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -266,4 +268,20 @@ public TransferSummary jobMonitor(@RequestParam String userEmail, @RequestParam summary.setTransferStatus(jobData.get(0).getStatus()); return summary; } + + @GetMapping("/user_batchJobs") + public List getBatchJobBasedOnEmailAndType(@RequestParam String userEmail, + @RequestParam String type) { + + List batchJobDetails = new ArrayList<>(); + Preconditions.checkNotNull(userEmail); + Preconditions.checkNotNull(type); + log.info(userEmail); + log.info(type); + if (validateUserEmail(userEmail)) { + batchJobDetails = jobService.getBatchJobDetailsBasedOnEmailAndType(userEmail, type); + } + + return batchJobDetails; + } } diff --git a/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java b/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java index 8b11186..2872316 100644 --- a/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java +++ b/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java @@ -4,7 +4,9 @@ import org.onedatashare.odsmetadata.entity.BatchJobExecutionParams; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import java.util.Date; @@ -27,4 +29,13 @@ public interface BatchJobRepository extends PagingAndSortingRepository findAllByBatchJobParams_ParameterValueLike(String userId, Pageable pr); List findByStartTimeIsGreaterThanEqualAndEndTimeIsLessThanEqualAndBatchJobParams_ParameterValueLike(Date startTime, Date endTime, String userId, Pageable pr); + + @Query("SELECT DISTINCT j FROM BatchJobExecution j " + + "INNER JOIN j.batchJobParams p " + + "INNER JOIN j.batchJobParams p2 " + + "WHERE ((p.parameterName = 'sourceCredentialType' AND p.parameterValue LIKE %:type%) " + + "OR (p.parameterName = 'destCredentialType' AND p.parameterValue LIKE %:type%)) " + + "AND p2.parameterValue LIKE %:email%") + List findBatchJobExecutionByEmailAndSourceAndDestinationType( + @Param("email") String email, @Param("type") String type); } diff --git a/src/main/java/org/onedatashare/odsmetadata/services/JobService.java b/src/main/java/org/onedatashare/odsmetadata/services/JobService.java index db663f8..5e59541 100644 --- a/src/main/java/org/onedatashare/odsmetadata/services/JobService.java +++ b/src/main/java/org/onedatashare/odsmetadata/services/JobService.java @@ -134,6 +134,15 @@ public Page getUserJobsByDateRange(String userId, Instant from, In return new PageImpl(batchJobDataList, pr, batchJobDataList.size()); } + public List getBatchJobDetailsBasedOnEmailAndType(String userEmail, String type){ + + List batchJobDetails = batchJobRepository + .findBatchJobExecutionByEmailAndSourceAndDestinationType(userEmail, type); + log.info("test : {}", batchJobDetails); + return batchJobDetails; + + } + private void processBatchJobExecutionData(List batchJobDataList, List batchJobExecutions) { if (!CollectionUtils.isEmpty(batchJobExecutions)) { batchJobExecutions.stream().forEach(batchJobExecution -> { From e30a8397f84d1557d89362fd2fc8cf8d81bda0db Mon Sep 17 00:00:00 2001 From: pratik08sha Date: Fri, 22 Dec 2023 00:21:44 -0500 Subject: [PATCH 3/3] batch jobs based on parameter map --- .../controller/BatchJobController.java | 19 ++++++++++------ .../repository/BatchJobRepository.java | 2 ++ .../odsmetadata/services/JobService.java | 22 ++++++++++++++++++- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java b/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java index 9437ea6..cdfee41 100644 --- a/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java +++ b/src/main/java/org/onedatashare/odsmetadata/controller/BatchJobController.java @@ -15,17 +15,12 @@ import org.springframework.data.domain.Pageable; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.onedatashare.odsmetadata.entity.BatchJobExecution; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; import java.util.regex.Pattern; /** @@ -284,4 +279,14 @@ public List getBatchJobBasedOnEmailAndType(@RequestParam Stri return batchJobDetails; } + + @GetMapping("/jobs/param_map") + public List getDistinctBatchJobsByParameterValues(@RequestBody HashMap paramMap) { + + log.info("test values : {}", paramMap); + List batchJobDetails = new ArrayList<>(); + Preconditions.checkNotNull(paramMap); + batchJobDetails = jobService.getBatchJobExecutionByParameterMap(paramMap); + return batchJobDetails; + } } diff --git a/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java b/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java index 2872316..194d4d0 100644 --- a/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java +++ b/src/main/java/org/onedatashare/odsmetadata/repository/BatchJobRepository.java @@ -30,6 +30,8 @@ public interface BatchJobRepository extends PagingAndSortingRepository findByStartTimeIsGreaterThanEqualAndEndTimeIsLessThanEqualAndBatchJobParams_ParameterValueLike(Date startTime, Date endTime, String userId, Pageable pr); + List findByBatchJobParams_ParameterValueIn(List paramValues); + @Query("SELECT DISTINCT j FROM BatchJobExecution j " + "INNER JOIN j.batchJobParams p " + "INNER JOIN j.batchJobParams p2 " + diff --git a/src/main/java/org/onedatashare/odsmetadata/services/JobService.java b/src/main/java/org/onedatashare/odsmetadata/services/JobService.java index 5e59541..23775b5 100644 --- a/src/main/java/org/onedatashare/odsmetadata/services/JobService.java +++ b/src/main/java/org/onedatashare/odsmetadata/services/JobService.java @@ -138,11 +138,31 @@ public List getBatchJobDetailsBasedOnEmailAndType(String user List batchJobDetails = batchJobRepository .findBatchJobExecutionByEmailAndSourceAndDestinationType(userEmail, type); - log.info("test : {}", batchJobDetails); return batchJobDetails; } + public List getBatchJobExecutionByParameterMap(Map paramMap){ + + List paramValues = paramMap.values().stream() + .filter(value -> value != null && !value.isEmpty()) + .collect(Collectors.toList()); + List batchJobExecutions = batchJobRepository.findByBatchJobParams_ParameterValueIn(paramValues); + List filteredJobs = batchJobExecutions.stream() + .filter(job -> matchesParameterMap(job, paramMap)) + .collect(Collectors.toList()); + return filteredJobs; + } + + private boolean matchesParameterMap(BatchJobExecution job, Map paramMap) { + + List jobParams = job.getBatchJobParams(); + + return paramMap.entrySet().stream().allMatch(entry -> jobParams.stream() + .anyMatch(param -> param.getParameterName().equals(entry.getKey()) + && param.getParameterValue().equals(entry.getValue()))); + } + private void processBatchJobExecutionData(List batchJobDataList, List batchJobExecutions) { if (!CollectionUtils.isEmpty(batchJobExecutions)) { batchJobExecutions.stream().forEach(batchJobExecution -> {