diff --git a/pom.xml b/pom.xml index e63aa14d53..b44eb8cac1 100644 --- a/pom.xml +++ b/pom.xml @@ -809,6 +809,18 @@ 9.2.1 + + org.opensearch.client + opensearch-rest-client + 3.3.2 + + + + org.opensearch.client + opensearch-java + 3.3.0 + + org.webjars swagger-ui diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java b/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java index d228286922..5acaaa863e 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java @@ -46,6 +46,7 @@ import org.broadinstitute.consent.http.health.ElasticSearchHealthCheck; import org.broadinstitute.consent.http.health.GCSHealthCheck; import org.broadinstitute.consent.http.health.OntologyHealthCheck; +import org.broadinstitute.consent.http.health.OpenSearchHealthCheck; import org.broadinstitute.consent.http.health.SamHealthCheck; import org.broadinstitute.consent.http.health.SendGridHealthCheck; import org.broadinstitute.consent.http.models.AuthUser; @@ -96,6 +97,7 @@ public class ConsentApplication extends Application { public static final String GCS_CHECK = "google-cloud-storage"; public static final String ES_CHECK = "elastic-search"; + public static final String OS_CHECK = "opensearch"; public static final String ONTOLOGY_CHECK = "ontology"; public static final String SAM_CHECK = "sam"; public static final String SG_CHECK = "sendgrid"; @@ -146,6 +148,7 @@ public void run(ConsentConfiguration config, Environment env) { // Health Checks env.healthChecks().register(GCS_CHECK, injector.getInstance(GCSHealthCheck.class)); env.healthChecks().register(ES_CHECK, injector.getInstance(ElasticSearchHealthCheck.class)); + env.healthChecks().register(OS_CHECK, injector.getInstance(OpenSearchHealthCheck.class)); env.healthChecks().register(ONTOLOGY_CHECK, injector.getInstance(OntologyHealthCheck.class)); env.healthChecks().register(SAM_CHECK, injector.getInstance(SamHealthCheck.class)); env.healthChecks().register(SG_CHECK, injector.getInstance(SendGridHealthCheck.class)); diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java index bb701266c1..0eed13b630 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java @@ -17,6 +17,7 @@ import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; import org.broadinstitute.consent.http.configurations.GoogleOAuth2Config; import org.broadinstitute.consent.http.configurations.MailConfiguration; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; import org.broadinstitute.consent.http.configurations.ServicesConfiguration; import org.broadinstitute.consent.http.db.AcknowledgementDAO; import org.broadinstitute.consent.http.db.CounterDAO; @@ -65,6 +66,7 @@ import org.broadinstitute.consent.http.service.NihService; import org.broadinstitute.consent.http.service.OidcService; import org.broadinstitute.consent.http.service.OntologyService; +import org.broadinstitute.consent.http.service.OpenSearchService; import org.broadinstitute.consent.http.service.SupportRequestService; import org.broadinstitute.consent.http.service.UseRestrictionConverter; import org.broadinstitute.consent.http.service.UserService; @@ -81,6 +83,7 @@ import org.broadinstitute.consent.http.service.dao.VoteServiceDAO; import org.broadinstitute.consent.http.service.feature.InstitutionAndLibraryCardEnforcement; import org.broadinstitute.consent.http.service.ontology.ElasticSearchSupport; +import org.broadinstitute.consent.http.service.ontology.OpenSearchSupport; import org.broadinstitute.consent.http.service.sam.SamService; import org.broadinstitute.consent.http.util.HttpClientUtil; import org.broadinstitute.consent.http.util.gson.GsonUtil; @@ -208,6 +211,11 @@ ElasticSearchConfiguration providesElasticSearchConfiguration() { return config.getElasticSearchConfiguration(); } + @Provides + OpenSearchConfiguration providesOpenSearchConfiguration() { + return config.getOpenSearchConfiguration(); + } + @Provides MailConfiguration providesMailConfiguration() { return config.getMailConfiguration(); @@ -323,6 +331,7 @@ DatasetService providesDatasetService() { providesDaaDAO(), providesDacDAO(), providesElasticSearchService(), + providesOpenSearchService(), providesEmailService(), providesOntologyService(), providesStudyDAO(), @@ -412,6 +421,7 @@ VoteService providesVoteService() { providesElectionDAO(), providesEmailService(), providesElasticSearchService(), + providesOpenSearchService(), providesUseRestrictionConverter(), providesVoteDAO(), providesVoteServiceDAO()); @@ -487,6 +497,20 @@ ElasticSearchService providesElasticSearchService() { providesStudyDAO()); } + @Provides + OpenSearchService providesOpenSearchService() { + return new OpenSearchService( + OpenSearchSupport.createRestClient(config.getOpenSearchConfiguration()), + config.getOpenSearchConfiguration(), + providesDacDAO(), + providesUserDAO(), + providesOntologyService(), + providesInstitutionDAO(), + providesDatasetDAO(), + providesDatasetServiceDAO(), + providesStudyDAO()); + } + @Provides UserDAO providesUserDAO() { return userDAO; @@ -584,6 +608,7 @@ DatasetRegistrationService providesDatasetRegistrationService() { providesDatasetServiceDAO(), providesGCSService(), providesElasticSearchService(), + providesOpenSearchService(), providesStudyDAO(), providesEmailService()); } diff --git a/src/main/java/org/broadinstitute/consent/http/configurations/ConsentConfiguration.java b/src/main/java/org/broadinstitute/consent/http/configurations/ConsentConfiguration.java index 53b0e6dba5..a1de3f94ba 100644 --- a/src/main/java/org/broadinstitute/consent/http/configurations/ConsentConfiguration.java +++ b/src/main/java/org/broadinstitute/consent/http/configurations/ConsentConfiguration.java @@ -38,6 +38,9 @@ public JerseyClientConfiguration getJerseyClientConfiguration() { @Valid @NotNull @JsonProperty private final ElasticSearchConfiguration elasticSearch = new ElasticSearchConfiguration(); + @Valid @NotNull @JsonProperty + private final OpenSearchConfiguration openSearch = new OpenSearchConfiguration(); + @Valid @NotNull @JsonProperty private final OidcConfiguration oidcConfiguration = new OidcConfiguration(); @@ -73,6 +76,10 @@ public ElasticSearchConfiguration getElasticSearchConfiguration() { return elasticSearch; } + public OpenSearchConfiguration getOpenSearchConfiguration() { + return openSearch; + } + public OidcConfiguration getOidcConfiguration() { return oidcConfiguration; } diff --git a/src/main/java/org/broadinstitute/consent/http/configurations/OpenSearchConfiguration.java b/src/main/java/org/broadinstitute/consent/http/configurations/OpenSearchConfiguration.java new file mode 100644 index 0000000000..7c123046b6 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/configurations/OpenSearchConfiguration.java @@ -0,0 +1,50 @@ +package org.broadinstitute.consent.http.configurations; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.validation.constraints.NotNull; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class OpenSearchConfiguration { + + @NotNull private String indexName; + + @NotNull private List servers; + + @NotNull private String datasetIndexName; + + /** This is configurable for testing purposes */ + private int port = 9201; + + public List getServers() { + return servers; + } + + public void setServers(List servers) { + this.servers = servers; + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getDatasetIndexName() { + return datasetIndexName; + } + + public void setDatasetIndexName(String datasetIndexName) { + this.datasetIndexName = datasetIndexName; + } +} diff --git a/src/main/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheck.java b/src/main/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheck.java new file mode 100644 index 0000000000..febec483c5 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheck.java @@ -0,0 +1,66 @@ +package org.broadinstitute.consent.http.health; + +import static org.broadinstitute.consent.http.service.ontology.OpenSearchSupport.jsonHeader; + +import com.codahale.metrics.health.HealthCheck; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.Inject; +import io.dropwizard.lifecycle.Managed; +import java.io.IOException; +import java.nio.charset.Charset; +import org.apache.commons.io.IOUtils; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; +import org.broadinstitute.consent.http.service.ontology.OpenSearchSupport; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; + +public class OpenSearchHealthCheck extends HealthCheck implements Managed { + + private final RestClient client; + + @Override + public void start() throws Exception {} + + @Override + public void stop() throws Exception { + if (client != null) { + client.close(); + } + } + + @Inject + public OpenSearchHealthCheck(OpenSearchConfiguration config) { + this.client = OpenSearchSupport.createRestClient(config); + } + + @Override + protected Result check() throws Exception { + try { + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.addHeader(jsonHeader.getName(), jsonHeader.getValue()); + Request request = new Request("GET", OpenSearchSupport.getClusterHealthPath()); + request.setOptions(builder.build()); + Response esResponse = client.performRequest(request); + if (esResponse.getStatusLine().getStatusCode() != 200) { + return Result.unhealthy( + "Invalid health check request: " + esResponse.getStatusLine().getReasonPhrase()); + } + String stringResponse = + IOUtils.toString(esResponse.getEntity().getContent(), Charset.defaultCharset()); + JsonObject jsonResponse = JsonParser.parseString(stringResponse).getAsJsonObject(); + String status = jsonResponse.get("status").getAsString(); + if (status.equalsIgnoreCase("red")) { + return Result.unhealthy("ClusterHealth is RED\n" + jsonResponse); + } + if (status.equalsIgnoreCase("yellow")) { + return Result.healthy("ClusterHealth is YELLOW\n" + jsonResponse); + } + } catch (IOException e) { + return Result.unhealthy("Unable to connect to OpenSearch"); + } + return Result.healthy("ClusterHealth is GREEN"); + } +} diff --git a/src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java b/src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java index 37675fbb2a..5fd074e73b 100644 --- a/src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java +++ b/src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java @@ -56,6 +56,7 @@ import org.broadinstitute.consent.http.service.DatasetRegistrationService; import org.broadinstitute.consent.http.service.DatasetService; import org.broadinstitute.consent.http.service.ElasticSearchService; +import org.broadinstitute.consent.http.service.OpenSearchService; import org.broadinstitute.consent.http.service.TDRService; import org.broadinstitute.consent.http.service.UserService; import org.broadinstitute.consent.http.util.JsonSchemaUtil; @@ -72,6 +73,7 @@ public class DatasetResource extends Resource { private final TDRService tdrService; private final UserService userService; private final ElasticSearchService elasticSearchService; + private final OpenSearchService openSearchService; private final JsonSchemaUtil jsonSchemaUtil; private final GCSService gcsService; @@ -82,6 +84,7 @@ public DatasetResource( UserService userService, DatasetRegistrationService datasetRegistrationService, ElasticSearchService elasticSearchService, + OpenSearchService openSearchService, TDRService tdrService, GCSService gcsService) { this.datasetService = datasetService; @@ -89,6 +92,7 @@ public DatasetResource( this.datasetRegistrationService = datasetRegistrationService; this.gcsService = gcsService; this.elasticSearchService = elasticSearchService; + this.openSearchService = openSearchService; this.tdrService = tdrService; this.jsonSchemaUtil = new JsonSchemaUtil(); } @@ -222,6 +226,7 @@ public Response patchByDatasetUpdate( } Dataset patched = datasetRegistrationService.patchDataset(datasetId, user, patch); elasticSearchService.synchronizeDatasetInESIndex(patched, user, false); + openSearchService.synchronizeDatasetInESIndex(patched, user, false); return Response.ok(patched).build(); } catch (Exception e) { return createExceptionResponse(e); @@ -376,7 +381,12 @@ public Response delete(@Auth DuosUser duosUser, @PathParam("datasetId") Integer } try (var deleteResponse = elasticSearchService.deleteIndex(datasetId, user.getUserId())) { if (!HttpStatusCodes.isSuccess(deleteResponse.getStatus())) { - logWarn("Unable to delete index for dataset: " + datasetId); + logWarn("Unable to delete Elasticsearch index for dataset: " + datasetId); + } + } + try (var deleteResponse = openSearchService.deleteIndex(datasetId, user.getUserId())) { + if (!HttpStatusCodes.isSuccess(deleteResponse.getStatus())) { + logWarn("Unable to delete OpenSearch index for dataset: " + datasetId); } } return Response.ok().build(); @@ -392,8 +402,10 @@ public Response indexDatasets(@Auth AuthUser authUser) { try { User user = userService.findUserByEmail(authUser.getEmail()); var datasetIds = datasetService.findAllDatasetIds(); - StreamingOutput indexResponse = elasticSearchService.indexDatasetIds(datasetIds, user); - return Response.ok(indexResponse, MediaType.APPLICATION_JSON).build(); + StreamingOutput elasticSearchResponse = + elasticSearchService.indexDatasetIds(datasetIds, user); + StreamingOutput openSearchResponse = openSearchService.indexDatasetIds(datasetIds, user); + return Response.ok(elasticSearchResponse, MediaType.APPLICATION_JSON).build(); } catch (Exception e) { return createExceptionResponse(e); } @@ -405,7 +417,9 @@ public Response indexDatasets(@Auth AuthUser authUser) { public Response indexDataset(@Auth AuthUser authUser, @PathParam("datasetId") Integer datasetId) { try { User user = userService.findUserByEmail(authUser.getEmail()); - return elasticSearchService.indexDataset(datasetId, user); + Response elasticSearchResponse = elasticSearchService.indexDataset(datasetId, user); + Response openSearchResponse = openSearchService.indexDataset(datasetId, user); + return elasticSearchResponse; } catch (Exception e) { return createExceptionResponse(e); } @@ -446,7 +460,7 @@ public Response searchDatasetIndex(@Auth DuosUser duosUser, String query) { @Timed public Response searchDatasetIndexStream(@Auth DuosUser duosUser, String query) { try { - InputStream inputStream = elasticSearchService.searchDatasetsStream(query); + InputStream inputStream = openSearchService.searchDatasetsStream(query); StreamingOutput stream = createStreamingOutput(inputStream); return Response.ok(stream).build(); } catch (Exception e) { diff --git a/src/main/java/org/broadinstitute/consent/http/service/DatasetRegistrationService.java b/src/main/java/org/broadinstitute/consent/http/service/DatasetRegistrationService.java index 7f0557d3cd..d176c383f4 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/DatasetRegistrationService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/DatasetRegistrationService.java @@ -56,6 +56,7 @@ public class DatasetRegistrationService implements ConsentLogger { private final DatasetServiceDAO datasetServiceDAO; private final GCSService gcsService; private final ElasticSearchService elasticSearchService; + private final OpenSearchService openSearchService; private final StudyDAO studyDAO; private final EmailService emailService; @@ -65,6 +66,7 @@ public DatasetRegistrationService( DatasetServiceDAO datasetServiceDAO, GCSService gcsService, ElasticSearchService elasticSearchService, + OpenSearchService openSearchService, StudyDAO studyDAO, EmailService emailService) { this.datasetDAO = datasetDAO; @@ -72,6 +74,7 @@ public DatasetRegistrationService( this.datasetServiceDAO = datasetServiceDAO; this.gcsService = gcsService; this.elasticSearchService = elasticSearchService; + this.openSearchService = openSearchService; this.studyDAO = studyDAO; this.emailService = emailService; } @@ -211,6 +214,15 @@ public List createDatasetsFromRegistration( } catch (Exception e) { logException(e); } + try (Response response = openSearchService.indexDatasets(createdDatasetIds, user)) { + if (response.getStatus() >= 400) { + logWarn( + String.format( + "Error indexing datasets from registration: %s", registration.getStudyName())); + } + } catch (Exception e) { + logException(e); + } return datasets; } @@ -280,6 +292,7 @@ public Dataset updateDataset( Dataset updatedDataset = datasetDAO.findDatasetById(datasetId); elasticSearchService.synchronizeDatasetInESIndex(updatedDataset, user, false); + openSearchService.synchronizeDatasetInESIndex(updatedDataset, user, false); return updatedDataset; } diff --git a/src/main/java/org/broadinstitute/consent/http/service/DatasetService.java b/src/main/java/org/broadinstitute/consent/http/service/DatasetService.java index 709433c249..e8dd6e8914 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/DatasetService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/DatasetService.java @@ -54,6 +54,7 @@ public class DatasetService implements ConsentLogger { private final DaaDAO daaDAO; private final DacDAO dacDAO; private final ElasticSearchService elasticSearchService; + private final OpenSearchService openSearchService; private final EmailService emailService; private final OntologyService ontologyService; private final StudyDAO studyDAO; @@ -67,6 +68,7 @@ public DatasetService( DaaDAO daaDAO, DacDAO dacDAO, ElasticSearchService elasticSearchService, + OpenSearchService openSearchService, EmailService emailService, OntologyService ontologyService, StudyDAO studyDAO, @@ -77,6 +79,7 @@ public DatasetService( this.daaDAO = daaDAO; this.dacDAO = dacDAO; this.elasticSearchService = elasticSearchService; + this.openSearchService = openSearchService; this.emailService = emailService; this.ontologyService = ontologyService; this.studyDAO = studyDAO; @@ -268,6 +271,7 @@ public Dataset updateDatasetDataUse(User user, Integer datasetId, DataUse dataUs } datasetDAO.updateDatasetDataUse(datasetId, dataUse.toString()); elasticSearchService.synchronizeDatasetInESIndex(d, user, false); + openSearchService.synchronizeDatasetInESIndex(d, user, false); return datasetDAO.findDatasetById(datasetId); } @@ -281,6 +285,7 @@ public Dataset syncDatasetDataUseTranslation(Integer datasetId, User user) { ontologyService.translateDataUse(dataset.getDataUse(), DataUseTranslationType.DATASET); datasetDAO.updateDatasetTranslatedDataUse(datasetId, translation); elasticSearchService.synchronizeDatasetInESIndex(dataset, user, false); + openSearchService.synchronizeDatasetInESIndex(dataset, user, false); return datasetDAO.findDatasetById(datasetId); } @@ -289,7 +294,16 @@ public void deleteDataset(Integer datasetId, Integer userId) throws Exception { if (dataset != null) { try (var response = elasticSearchService.deleteIndex(datasetId, userId)) { if (!HttpStatusCodes.isSuccess(response.getStatus())) { - logWarn("Response error, unable to delete dataset from index: %s".formatted(datasetId)); + logWarn( + "Response error, unable to delete dataset from Elasticsearch index: %s" + .formatted(datasetId)); + } + } + try (var response = openSearchService.deleteIndex(datasetId, userId)) { + if (!HttpStatusCodes.isSuccess(response.getStatus())) { + logWarn( + "Response error, unable to delete dataset from OpenSearch index: %s" + .formatted(datasetId)); } } datasetServiceDAO.deleteDataset(dataset, userId); @@ -304,7 +318,16 @@ public void deleteStudy(Study study, User user) throws Exception { try (var response = elasticSearchService.deleteIndex(datasetId, user.getUserId())) { if (!HttpStatusCodes.isSuccess(response.getStatus())) { logWarn( - "Response error, unable to delete dataset from index: %s" + "Response error, unable to delete dataset from Elasticsearch index: %s" + .formatted(datasetId)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + try (var response = openSearchService.deleteIndex(datasetId, user.getUserId())) { + if (!HttpStatusCodes.isSuccess(response.getStatus())) { + logWarn( + "Response error, unable to delete dataset from OpenSearch index: %s" .formatted(datasetId)); } } catch (IOException e) { @@ -333,6 +356,7 @@ public Dataset approveDataset(Dataset dataset, User user, Boolean approval) { if (currentApprovalState == null || !currentApprovalState) { datasetDAO.updateDatasetApproval(approval, Instant.now(), user.getUserId(), datasetId); elasticSearchService.asyncDatasetInESIndex(datasetId, user, true); + openSearchService.asyncDatasetInESIndex(datasetId, user, true); datasetReturn = datasetDAO.findDatasetWithoutFSOInformation(datasetId); } else { if (approval == null || !approval) { @@ -442,6 +466,7 @@ public Study convertDatasetToStudy(User user, Dataset dataset, StudyConversion s datasetDAO.updateDatasetName(dataset.getDatasetId(), studyConversion.getDatasetName()); } elasticSearchService.synchronizeDatasetInESIndex(dataset, user, false); + openSearchService.synchronizeDatasetInESIndex(dataset, user, false); List dictionaries = datasetDAO.getDictionaryTerms(); // Handle "Phenotype/Indication" if (studyConversion.getPhenotype() != null) { @@ -524,7 +549,10 @@ public Study updateStudyCustodians(User user, Integer studyId, String custodians } List datasets = datasetDAO.findDatasetsByIdList(study.getDatasetIds()); datasets.forEach( - dataset -> elasticSearchService.synchronizeDatasetInESIndex(dataset, user, false)); + dataset -> { + elasticSearchService.synchronizeDatasetInESIndex(dataset, user, false); + openSearchService.synchronizeDatasetInESIndex(dataset, user, false); + }); return studyDAO.findStudyById(studyId); } @@ -723,7 +751,11 @@ public Study patchStudy(Integer studyId, User user, StudyPatch patch) { datasetServiceDAO.patchStudy(study, user, patch); study .getDatasetIds() - .forEach(datasetId -> elasticSearchService.asyncDatasetInESIndex(datasetId, user, true)); + .forEach( + datasetId -> { + elasticSearchService.asyncDatasetInESIndex(datasetId, user, true); + openSearchService.asyncDatasetInESIndex(datasetId, user, true); + }); return studyDAO.findStudyById(studyId); } catch (Exception ex) { logException(ex); diff --git a/src/main/java/org/broadinstitute/consent/http/service/OpenSearchService.java b/src/main/java/org/broadinstitute/consent/http/service/OpenSearchService.java new file mode 100644 index 0000000000..626131511f --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/service/OpenSearchService.java @@ -0,0 +1,533 @@ +package org.broadinstitute.consent.http.service; + +import com.google.api.client.http.HttpStatusCodes; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.JsonArray; +import jakarta.ws.rs.HttpMethod; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; +import org.broadinstitute.consent.http.db.DacDAO; +import org.broadinstitute.consent.http.db.DatasetDAO; +import org.broadinstitute.consent.http.db.InstitutionDAO; +import org.broadinstitute.consent.http.db.StudyDAO; +import org.broadinstitute.consent.http.db.UserDAO; +import org.broadinstitute.consent.http.models.Dac; +import org.broadinstitute.consent.http.models.Dataset; +import org.broadinstitute.consent.http.models.DatasetProperty; +import org.broadinstitute.consent.http.models.Institution; +import org.broadinstitute.consent.http.models.Study; +import org.broadinstitute.consent.http.models.StudyProperty; +import org.broadinstitute.consent.http.models.User; +import org.broadinstitute.consent.http.models.elastic_search.DacTerm; +import org.broadinstitute.consent.http.models.elastic_search.DatasetTerm; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchHits; +import org.broadinstitute.consent.http.models.elastic_search.InstitutionTerm; +import org.broadinstitute.consent.http.models.elastic_search.StudyTerm; +import org.broadinstitute.consent.http.models.elastic_search.UserTerm; +import org.broadinstitute.consent.http.models.ontology.DataUseSummary; +import org.broadinstitute.consent.http.service.dao.DatasetServiceDAO; +import org.broadinstitute.consent.http.util.ConsentLogger; +import org.broadinstitute.consent.http.util.ThreadUtils; +import org.broadinstitute.consent.http.util.gson.GsonUtil; +import org.opensearch.client.Request; +import org.opensearch.client.RestClient; + +public class OpenSearchService implements ConsentLogger { + + private final ExecutorService executorService = + new ThreadUtils().getExecutorService(OpenSearchService.class); + private final RestClient esClient; + private final OpenSearchConfiguration esConfig; + private final DacDAO dacDAO; + private final UserDAO userDAO; + private final OntologyService ontologyService; + private final InstitutionDAO institutionDAO; + private final DatasetDAO datasetDAO; + private final DatasetServiceDAO datasetServiceDAO; + private final StudyDAO studyDAO; + + public OpenSearchService( + RestClient esClient, + OpenSearchConfiguration esConfig, + DacDAO dacDAO, + UserDAO userDao, + OntologyService ontologyService, + InstitutionDAO institutionDAO, + DatasetDAO datasetDAO, + DatasetServiceDAO datasetServiceDAO, + StudyDAO studyDAO) { + this.esClient = esClient; + this.esConfig = esConfig; + this.dacDAO = dacDAO; + this.userDAO = userDao; + this.ontologyService = ontologyService; + this.institutionDAO = institutionDAO; + this.datasetDAO = datasetDAO; + this.datasetServiceDAO = datasetServiceDAO; + this.studyDAO = studyDAO; + } + + private static final int MAX_RESULT_WINDOW = 10000; + + private static final String BULK_HEADER = + """ + { "index": {"_index": "dataset", "_id": "%d"} } + """; + + private static final String DELETE_QUERY = + """ + { "query": { "bool": { "must": [ { "match": { "_index": "dataset" } }, { "match": { "_id": "%d" } } ] } } } + """; + + private Response performRequest(Request request) throws IOException { + var response = esClient.performRequest(request); + var status = response.getStatusLine().getStatusCode(); + if (status != 200) { + throw new IOException("Invalid Opensearch query"); + } + var body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + return Response.status(status).entity(body).build(); + } + + public Response indexDatasetTerms(List datasets, User user) throws IOException { + List bulkApiCall = new ArrayList<>(); + + datasets.forEach( + dsTerm -> { + bulkApiCall.add(BULK_HEADER.formatted(dsTerm.getDatasetId())); + bulkApiCall.add(GsonUtil.getInstance().toJson(dsTerm) + "\n"); + updateDatasetIndexDate(dsTerm.getDatasetId(), user.getUserId(), Instant.now()); + }); + + Request bulkRequest = + new Request(HttpMethod.PUT, "/" + esConfig.getDatasetIndexName() + "/_bulk"); + + bulkRequest.setEntity( + new StringEntity(String.join("", bulkApiCall) + "\n", ContentType.APPLICATION_JSON)); + + return performRequest(bulkRequest); + } + + public Response deleteIndex(Integer datasetId, Integer userId) throws IOException { + Request deleteRequest = + new Request(HttpMethod.POST, "/" + esConfig.getDatasetIndexName() + "/_delete_by_query"); + deleteRequest.setEntity( + new StringEntity(DELETE_QUERY.formatted(datasetId), ContentType.APPLICATION_JSON)); + updateDatasetIndexDate(datasetId, userId, null); + return performRequest(deleteRequest); + } + + public boolean invalidResultWindow(String query) { + try { + var queryJson = GsonUtil.getInstance().fromJson(query, Map.class); + + long size = (long) queryJson.getOrDefault("size", 10L); + long from = (long) queryJson.getOrDefault("from", 0L); + + return from + size > MAX_RESULT_WINDOW; + } catch (Exception e) { + logWarn("Unable to parse query for result window validation: " + e.getMessage()); + return true; + } + } + + public boolean validateQuery(String query) throws IOException { + if (invalidResultWindow(query)) { + return false; + } + + // Remove `sort`, `size` and `from` parameters from query, otherwise validation will fail + var modifiedQuery = + query + .replaceAll("\"sort\": ?\\[(.*?)\\],?", "") + .replaceAll("\"size\": ?\\d+,?", "") + .replaceAll("\"from\": ?\\d+,?", ""); + + Request validateRequest = + new Request(HttpMethod.GET, "/" + esConfig.getDatasetIndexName() + "/_validate/query"); + validateRequest.setEntity(new StringEntity(modifiedQuery, ContentType.APPLICATION_JSON)); + Response response = performRequest(validateRequest); + + var entity = response.getEntity().toString(); + var json = GsonUtil.getInstance().fromJson(entity, Map.class); + + return (boolean) json.get("valid"); + } + + public Response searchDatasets(String query) throws IOException { + if (!validateQuery(query)) { + throw new IOException("Invalid Opensearch query"); + } + + Request searchRequest = + new Request(HttpMethod.GET, "/" + esConfig.getDatasetIndexName() + "/_search"); + searchRequest.setEntity(new StringEntity(query, ContentType.APPLICATION_JSON)); + + Response response = performRequest(searchRequest); + + var entity = response.getEntity().toString(); + var json = GsonUtil.getInstance().fromJson(entity, ElasticSearchHits.class); + var hits = json.getHits(); + + return Response.ok().entity(hits).build(); + } + + public InputStream searchDatasetsStream(String query) throws IOException { + if (invalidResultWindow(query)) { + throw new IOException("Invalid Opensearch query"); + } + Request searchRequest = + new Request(HttpMethod.GET, "/" + esConfig.getDatasetIndexName() + "/_search"); + searchRequest.setEntity(new StringEntity(query, ContentType.APPLICATION_JSON)); + var response = esClient.performRequest(searchRequest); + var status = response.getStatusLine().getStatusCode(); + if (status != 200) { + throw new IOException("Invalid Opensearch query"); + } + return response.getEntity().getContent(); + } + + public StudyTerm toStudyTerm(Study study) { + if (Objects.isNull(study)) { + return null; + } + + StudyTerm term = new StudyTerm(); + + term.setDescription(study.getDescription()); + term.setStudyName(study.getName()); + term.setStudyId(study.getStudyId()); + term.setDataTypes(study.getDataTypes()); + term.setPiName(study.getPiName()); + term.setPublicVisibility(study.getPublicVisibility()); + + findStudyProperty(study.getProperties(), "dbGaPPhsID") + .ifPresent(prop -> term.setPhsId(prop.getValue().toString())); + + findStudyProperty(study.getProperties(), "phenotypeIndication") + .ifPresent(prop -> term.setPhenotype(prop.getValue().toString())); + + findStudyProperty(study.getProperties(), "species") + .ifPresent(prop -> term.setSpecies(prop.getValue().toString())); + + findStudyProperty(study.getProperties(), "dataCustodianEmail") + .ifPresent( + prop -> { + JsonArray jsonArray = (JsonArray) prop.getValue(); + List dataCustodianEmail = new ArrayList<>(); + jsonArray.forEach(email -> dataCustodianEmail.add(email.getAsString())); + term.setDataCustodianEmail(dataCustodianEmail); + }); + + if (Objects.nonNull(study.getCreateUserId())) { + term.setDataSubmitterId(study.getCreateUserId()); + User user = userDAO.findUserById(study.getCreateUserId()); + if (Objects.nonNull(user)) { + study.setCreateUserEmail(user.getEmail()); + } + } + + if (Objects.nonNull(study.getCreateUserEmail())) { + term.setDataSubmitterEmail(study.getCreateUserEmail()); + } + + findStudyProperty(study.getProperties(), "assets") + .ifPresent( + prop -> { + Object value = prop.getValue(); + Map assetsMap; + // When property is loaded from db it is deserialized as JsonObject + if (value instanceof com.google.gson.JsonElement) { + assetsMap = + GsonUtil.getInstance() + .fromJson( + (com.google.gson.JsonElement) value, + new com.google.gson.reflect.TypeToken< + Map>() {}.getType()); + // Otherwise Gson deserializes JSON and creates a LinkedTreeMap + } else if (value instanceof Map) { + assetsMap = (Map) value; + // Fallback: try to parse as JSON string + } else { + assetsMap = + GsonUtil.getInstance() + .fromJson( + value.toString(), + new com.google.gson.reflect.TypeToken< + Map>() {}.getType()); + } + term.setAssets(assetsMap); + }); + + return term; + } + + public UserTerm toUserTerm(User user) { + if (Objects.isNull(user)) { + return null; + } + InstitutionTerm institution = + (Objects.nonNull(user.getInstitutionId())) + ? toInstitutionTerm(institutionDAO.findInstitutionById(user.getInstitutionId())) + : null; + return new UserTerm(user.getUserId(), user.getDisplayName(), institution); + } + + public DacTerm toDacTerm(Dac dac) { + if (Objects.isNull(dac)) { + return null; + } + return new DacTerm(dac.getDacId(), dac.getName(), dac.getEmail()); + } + + public InstitutionTerm toInstitutionTerm(Institution institution) { + if (Objects.isNull(institution)) { + return null; + } + return new InstitutionTerm(institution.getId(), institution.getName()); + } + + public void asyncDatasetInESIndex(Integer datasetId, User user, boolean force) { + ListeningExecutorService listeningExecutorService = + MoreExecutors.listeningDecorator(executorService); + ListenableFuture syncFuture = + listeningExecutorService.submit( + () -> { + Dataset dataset = datasetDAO.findDatasetById(datasetId); + synchronizeDatasetInESIndex(dataset, user, force); + return dataset; + }); + Futures.addCallback( + syncFuture, + new FutureCallback<>() { + @Override + public void onSuccess(Dataset d) { + logInfo( + "Successfully synchronized dataset in ES index: %s" + .formatted(d.getDatasetIdentifier())); + } + + @Override + public void onFailure(Throwable t) { + logWarn( + "Failed to synchronize dataset in ES index: %s".formatted(datasetId) + + ": " + + t.getMessage()); + } + }, + listeningExecutorService); + } + + /** + * Synchronize the dataset in the ES index. This will only index the dataset if it has been + * previously indexed, UNLESS the force argument is true which means it will index the dataset and + * update the dataset's last indexed date value. + * + * @param dataset The Dataset + * @param user The User + * @param force Boolean to force the index update regardless of dataset's indexed date status. + */ + public void synchronizeDatasetInESIndex(Dataset dataset, User user, boolean force) { + if (force || dataset.getIndexedDate() != null) { + try (var response = indexDataset(dataset.getDatasetId(), user)) { + if (!HttpStatusCodes.isSuccess(response.getStatus())) { + logWarn("Response error, unable to index dataset: %s".formatted(dataset.getDatasetId())); + } + } catch (IOException e) { + logWarn("Exception, unable to index dataset: %s".formatted(dataset.getDatasetId())); + } + } + } + + public Response indexDataset(Integer datasetId, User user) throws IOException { + return indexDatasets(List.of(datasetId), user); + } + + public Response indexDatasets(List datasetIds, User user) throws IOException { + // Datasets in list context may not have their study populated, so we need to ensure that is + // true before trying to index them in ES. + List datasetTerms = + datasetIds.stream().map(datasetDAO::findDatasetById).map(this::toDatasetTerm).toList(); + return indexDatasetTerms(datasetTerms, user); + } + + /** + * Sequentially index datasets to OpenSearch by ID list. Note that this is intended for large + * lists of dataset ids. For small sets of datasets (i.e. <~25), it is efficient to index them in + * bulk using the {@link #indexDatasets(List, User)} method. + * + * @param datasetIds List of Dataset IDs to index + * @return StreamingOutput of OpenSearch responses from indexing datasets + */ + public StreamingOutput indexDatasetIds(List datasetIds, User user) { + Integer lastDatasetId = datasetIds.get(datasetIds.size() - 1); + return output -> { + output.write("[".getBytes()); + datasetIds.forEach( + id -> { + try (Response response = indexDataset(id, user)) { + output.write(response.getEntity().toString().getBytes()); + if (!id.equals(lastDatasetId)) { + output.write(",".getBytes()); + } + output.write("\n".getBytes()); + } catch (IOException e) { + logException("Error indexing dataset term for dataset id: %d ".formatted(id), e); + } + }); + output.write("]".getBytes()); + }; + } + + public Response indexStudy(Integer studyId, User user) { + Study study = studyDAO.findStudyById(studyId); + // The dao call above does not populate its datasets so we need to check for datasetIds + if (study != null && !study.getDatasetIds().isEmpty()) { + try (Response response = indexDatasets(study.getDatasetIds().stream().toList(), user)) { + return response; + } catch (Exception e) { + logException(String.format("Failed to index datasets for study id: %d", studyId), e); + return Response.status(Status.INTERNAL_SERVER_ERROR).build(); + } + } + return Response.status(Status.NOT_FOUND).build(); + } + + public DatasetTerm toDatasetTerm(Dataset dataset) { + if (Objects.isNull(dataset)) { + return null; + } + + DatasetTerm term = new DatasetTerm(); + + term.setDatasetId(dataset.getDatasetId()); + Optional.ofNullable(dataset.getCreateUserId()) + .ifPresent( + userId -> { + User user = userDAO.findUserById(dataset.getCreateUserId()); + term.setCreateUserId(dataset.getCreateUserId()); + term.setCreateUserDisplayName(user.getDisplayName()); + term.setSubmitter(toUserTerm(user)); + }); + Optional.ofNullable(dataset.getUpdateUserId()) + .map(userDAO::findUserById) + .map(this::toUserTerm) + .ifPresent(term::setUpdateUser); + term.setDatasetIdentifier(dataset.getDatasetIdentifier()); + term.setDeletable(dataset.getDeletable()); + term.setDatasetName(dataset.getName()); + + if (Objects.nonNull(dataset.getStudy())) { + term.setStudy(toStudyTerm(dataset.getStudy())); + } + + Optional.ofNullable(dataset.getDacId()) + .ifPresent( + dacId -> { + Dac dac = dacDAO.findById(dataset.getDacId()); + term.setDacId(dataset.getDacId()); + if (Objects.nonNull(dataset.getDacApproval())) { + term.setDacApproval(dataset.getDacApproval()); + } + term.setDac(toDacTerm(dac)); + }); + + if (Objects.nonNull(dataset.getDataUse())) { + DataUseSummary summary = ontologyService.translateDataUseSummary(dataset.getDataUse()); + if (summary != null) { + term.setDataUse(summary); + } else { + logWarn("No data use summary for dataset id: %d".formatted(dataset.getDatasetId())); + } + } + + Optional.ofNullable(dataset.getNihInstitutionalCertificationFile()) + .ifPresent(obj -> term.setHasInstitutionCertification(true)); + + findDatasetProperty(dataset.getProperties(), "accessManagement") + .ifPresent( + datasetProperty -> + term.setAccessManagement(datasetProperty.getPropertyValueAsString())); + + findFirstDatasetPropertyByName(dataset.getProperties(), "# of participants") + .ifPresent( + datasetProperty -> { + String value = datasetProperty.getPropertyValueAsString(); + try { + term.setParticipantCount(Integer.valueOf(value)); + } catch (NumberFormatException e) { + logWarn( + String.format( + "Unable to coerce participant count to integer: %s for dataset: %s", + value, dataset.getDatasetIdentifier())); + } + }); + + findDatasetProperty(dataset.getProperties(), "url") + .ifPresent(datasetProperty -> term.setUrl(datasetProperty.getPropertyValueAsString())); + + findDatasetProperty(dataset.getProperties(), "dataLocation") + .ifPresent( + datasetProperty -> term.setDataLocation(datasetProperty.getPropertyValueAsString())); + + return term; + } + + protected void updateDatasetIndexDate(Integer datasetId, Integer userId, Instant indexDate) { + // It is possible that a dataset has been deleted. If so, we don't want to try and update it. + Dataset dataset = datasetDAO.findDatasetById(datasetId); + if (dataset != null) { + try { + datasetServiceDAO.updateDatasetIndex(datasetId, userId, indexDate); + } catch (SQLException e) { + // We don't want to send these to Sentry, but we do want to log them for follow up off cycle + logWarn("Error updating dataset indexed date for dataset id: %d ".formatted(datasetId), e); + } + } + } + + Optional findDatasetProperty( + Collection props, String schemaProp) { + return (props == null) + ? Optional.empty() + : props.stream() + .filter(p -> Objects.nonNull(p.getSchemaProperty())) + .filter(p -> p.getSchemaProperty().equals(schemaProp)) + .findFirst(); + } + + Optional findFirstDatasetPropertyByName( + Collection props, String propertyName) { + return (props == null) + ? Optional.empty() + : props.stream() + .filter(p -> p.getPropertyName().equalsIgnoreCase(propertyName)) + .findFirst(); + } + + Optional findStudyProperty(Collection props, String key) { + return (props == null) + ? Optional.empty() + : props.stream().filter(p -> p.getKey().equals(key)).findFirst(); + } +} diff --git a/src/main/java/org/broadinstitute/consent/http/service/VoteService.java b/src/main/java/org/broadinstitute/consent/http/service/VoteService.java index 9020c6357e..f9be861610 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/VoteService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/VoteService.java @@ -57,6 +57,7 @@ public class VoteService implements ConsentLogger { private final ElectionDAO electionDAO; private final EmailService emailService; private final ElasticSearchService elasticSearchService; + private final OpenSearchService openSearchService; private final UseRestrictionConverter useRestrictionConverter; private final VoteDAO voteDAO; private final VoteServiceDAO voteServiceDAO; @@ -70,6 +71,7 @@ public VoteService( ElectionDAO electionDAO, EmailService emailService, ElasticSearchService elasticSearchService, + OpenSearchService openSearchService, UseRestrictionConverter useRestrictionConverter, VoteDAO voteDAO, VoteServiceDAO voteServiceDAO) { @@ -80,6 +82,7 @@ public VoteService( this.electionDAO = electionDAO; this.emailService = emailService; this.elasticSearchService = elasticSearchService; + this.openSearchService = openSearchService; this.useRestrictionConverter = useRestrictionConverter; this.voteDAO = voteDAO; this.voteServiceDAO = voteServiceDAO; @@ -260,6 +263,7 @@ public void sendDatasetApprovalNotifications(List votes, User user) { try { elasticSearchService.indexDatasets(datasetIds, user); + openSearchService.indexDatasets(datasetIds, user); } catch (Exception e) { logException("Error indexing datasets for approved DARs: " + e.getMessage(), e); } diff --git a/src/main/java/org/broadinstitute/consent/http/service/ontology/OpenSearchSupport.java b/src/main/java/org/broadinstitute/consent/http/service/ontology/OpenSearchSupport.java new file mode 100644 index 0000000000..5e97ce8efb --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/service/ontology/OpenSearchSupport.java @@ -0,0 +1,26 @@ +package org.broadinstitute.consent.http.service.ontology; + +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.message.BasicHeader; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; +import org.opensearch.client.RestClient; + +@SuppressWarnings("WeakerAccess") +public class OpenSearchSupport { + + public static RestClient createRestClient(OpenSearchConfiguration configuration) { + HttpHost[] hosts = + configuration.getServers().stream() + .map(server -> new HttpHost("http", server, configuration.getPort())) + .toList() + .toArray(new HttpHost[configuration.getServers().size()]); + return RestClient.builder(hosts).build(); + } + + public static String getClusterHealthPath() { + return "/_cluster/health"; + } + + public static Header jsonHeader = new BasicHeader("Content-Type", "application/json"); +} diff --git a/src/main/resources/assets/paths/datasetSearchIndexV2.yaml b/src/main/resources/assets/paths/datasetSearchIndexV2.yaml index 038d961bd9..754be4fe4d 100644 --- a/src/main/resources/assets/paths/datasetSearchIndexV2.yaml +++ b/src/main/resources/assets/paths/datasetSearchIndexV2.yaml @@ -22,7 +22,7 @@ post: bool: must: - match: - _type: dataset + _index: dataset - match: _id: 1440 example2: @@ -34,7 +34,7 @@ post: bool: must: - match: - _type: dataset + _index: dataset - exists: field: study tags: diff --git a/src/test/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheckTest.java b/src/test/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheckTest.java new file mode 100644 index 0000000000..ca5d4465e2 --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheckTest.java @@ -0,0 +1,76 @@ +package org.broadinstitute.consent.http.health; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +import com.codahale.metrics.health.HealthCheck; +import com.google.api.client.http.HttpStatusCodes; +import java.util.Collections; +import org.broadinstitute.consent.http.MockServerTestHelper; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class OpenSearchHealthCheckTest extends MockServerTestHelper { + + private OpenSearchHealthCheck healthCheck; + private OpenSearchConfiguration config; + + @BeforeEach + void init() { + config = new OpenSearchConfiguration(); + config.setServers(Collections.singletonList("localhost")); + config.setPort(CONTAINER.getServerPort()); + } + + private void initHealthCheck(String status, Integer statusCode) { + try { + String stringResponse = "{ \"status\": \"" + status + "\" }"; + mockServerClient + .when(request()) + .respond(response().withStatusCode(statusCode).withBody(stringResponse)); + + healthCheck = new OpenSearchHealthCheck(config); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + void testCheckSuccessGreen() throws Exception { + initHealthCheck("green", HttpStatusCodes.STATUS_CODE_OK); + + HealthCheck.Result result = healthCheck.check(); + assertTrue(result.isHealthy()); + } + + @Test + void testCheckSuccessYellow() throws Exception { + initHealthCheck("yellow", HttpStatusCodes.STATUS_CODE_OK); + + HealthCheck.Result result = healthCheck.check(); + assertTrue(result.isHealthy()); + } + + @Test + void testCheckFailureRed() throws Exception { + initHealthCheck("red", HttpStatusCodes.STATUS_CODE_OK); + + HealthCheck.Result result = healthCheck.check(); + assertFalse(result.isHealthy()); + } + + @Test + void testCheckServerFailure() throws Exception { + initHealthCheck("green", HttpStatusCodes.STATUS_CODE_SERVER_ERROR); + + HealthCheck.Result result = healthCheck.check(); + assertFalse(result.isHealthy()); + } +} diff --git a/src/test/java/org/broadinstitute/consent/http/resources/DatasetResourceTest.java b/src/test/java/org/broadinstitute/consent/http/resources/DatasetResourceTest.java index e7191660e1..2c8fca246e 100644 --- a/src/test/java/org/broadinstitute/consent/http/resources/DatasetResourceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/resources/DatasetResourceTest.java @@ -66,6 +66,7 @@ import org.broadinstitute.consent.http.service.DatasetRegistrationService; import org.broadinstitute.consent.http.service.DatasetService; import org.broadinstitute.consent.http.service.ElasticSearchService; +import org.broadinstitute.consent.http.service.OpenSearchService; import org.broadinstitute.consent.http.service.TDRService; import org.broadinstitute.consent.http.service.UserService; import org.broadinstitute.consent.http.util.gson.GsonUtil; @@ -86,6 +87,8 @@ class DatasetResourceTest extends AbstractTestHelper { @Mock private ElasticSearchService elasticSearchService; + @Mock private OpenSearchService openSearchService; + @Mock private TDRService tdrService; @Mock private UserService userService; @@ -107,6 +110,7 @@ void initResource() { userService, datasetRegistrationService, elasticSearchService, + openSearchService, tdrService, gcsService); } @@ -395,6 +399,7 @@ void testDeleteSuccessAdmin() throws Exception { user.addRole(UserRoles.Admin()); when(datasetService.findDatasetById(any(), any())).thenReturn(dataSet); when(elasticSearchService.deleteIndex(any(), any())).thenReturn(mockResponse); + when(openSearchService.deleteIndex(any(), any())).thenReturn(mockResponse); try (var response = resource.delete(duosUser, 1)) { assertEquals(HttpStatusCodes.STATUS_CODE_OK, response.getStatus()); @@ -413,6 +418,7 @@ void testDeleteSuccessChairperson() throws Exception { when(datasetService.findDatasetById(any(), any())).thenReturn(dataSet); when(elasticSearchService.deleteIndex(any(), any())).thenReturn(mockResponse); + when(openSearchService.deleteIndex(any(), any())).thenReturn(mockResponse); try (var response = resource.delete(duosUser, 1)) { assertEquals(HttpStatusCodes.STATUS_CODE_OK, response.getStatus()); @@ -562,7 +568,7 @@ void testSearchDatasetIndex() throws IOException { @Test void testSearchDatasetIndexStream() throws IOException { String query = "{ \"dataUse\": [\"HMB\"] }"; - when(elasticSearchService.searchDatasetsStream(any())) + when(openSearchService.searchDatasetsStream(any())) .thenReturn(IOUtils.toInputStream(query, Charset.defaultCharset())); try (var response = resource.searchDatasetIndexStream(duosUser, query)) { assertEquals(HttpStatusCodes.STATUS_CODE_OK, response.getStatus()); diff --git a/src/test/java/org/broadinstitute/consent/http/service/DatasetRegistrationServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/DatasetRegistrationServiceTest.java index 6d27a3612a..934e3ae3bf 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/DatasetRegistrationServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/DatasetRegistrationServiceTest.java @@ -85,6 +85,8 @@ class DatasetRegistrationServiceTest extends AbstractTestHelper { @Mock private ElasticSearchService elasticSearchService; + @Mock private OpenSearchService openSearchService; + @Mock private EmailService emailService; @BeforeEach @@ -96,6 +98,7 @@ void setUp() { datasetServiceDAO, gcsService, elasticSearchService, + openSearchService, studyDAO, emailService); } diff --git a/src/test/java/org/broadinstitute/consent/http/service/DatasetServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/DatasetServiceTest.java index 21c1ef8e6a..d7011e8e4b 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/DatasetServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/DatasetServiceTest.java @@ -73,6 +73,7 @@ class DatasetServiceTest extends AbstractTestHelper { @Mock private DaaDAO daaDAO; @Mock private DacDAO dacDAO; @Mock private ElasticSearchService elasticSearchService; + @Mock private OpenSearchService openSearchService; @Mock private EmailService emailService; @Mock private OntologyService ontologyService; @Mock private StudyDAO studyDAO; @@ -89,6 +90,7 @@ void initService() { daaDAO, dacDAO, elasticSearchService, + openSearchService, emailService, ontologyService, studyDAO, diff --git a/src/test/java/org/broadinstitute/consent/http/service/OpenSearchServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/OpenSearchServiceTest.java new file mode 100644 index 0000000000..6ffbd7cde9 --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/service/OpenSearchServiceTest.java @@ -0,0 +1,857 @@ +package org.broadinstitute.consent.http.service; + +import static jakarta.ws.rs.core.Response.Status.fromStatusCode; +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.client.http.HttpStatusCodes; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.io.IOUtils; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicHttpResponse; +import org.apache.hc.core5.http.message.StatusLine; +import org.broadinstitute.consent.http.AbstractTestHelper; +import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration; +import org.broadinstitute.consent.http.db.DacDAO; +import org.broadinstitute.consent.http.db.DatasetDAO; +import org.broadinstitute.consent.http.db.InstitutionDAO; +import org.broadinstitute.consent.http.db.StudyDAO; +import org.broadinstitute.consent.http.db.UserDAO; +import org.broadinstitute.consent.http.enumeration.PropertyType; +import org.broadinstitute.consent.http.models.Dac; +import org.broadinstitute.consent.http.models.DataAccessRequest; +import org.broadinstitute.consent.http.models.DataUse; +import org.broadinstitute.consent.http.models.DataUseBuilder; +import org.broadinstitute.consent.http.models.Dataset; +import org.broadinstitute.consent.http.models.DatasetProperty; +import org.broadinstitute.consent.http.models.FileStorageObject; +import org.broadinstitute.consent.http.models.Institution; +import org.broadinstitute.consent.http.models.LibraryCard; +import org.broadinstitute.consent.http.models.Study; +import org.broadinstitute.consent.http.models.StudyProperty; +import org.broadinstitute.consent.http.models.User; +import org.broadinstitute.consent.http.models.elastic_search.DatasetTerm; +import org.broadinstitute.consent.http.models.ontology.DataUseSummary; +import org.broadinstitute.consent.http.models.ontology.DataUseTerm; +import org.broadinstitute.consent.http.service.dao.DatasetServiceDAO; +import org.broadinstitute.consent.http.util.gson.GsonUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; + +@ExtendWith(MockitoExtension.class) +class OpenSearchServiceTest extends AbstractTestHelper { + + private OpenSearchService service; + + @Mock private RestClient esClient; + + @Mock private OntologyService ontologyService; + + @Mock private OpenSearchConfiguration esConfig; + + @Mock private DacDAO dacDAO; + + @Mock private UserDAO userDao; + + @Mock private InstitutionDAO institutionDAO; + + @Mock private DatasetDAO datasetDAO; + + @Mock private DatasetServiceDAO datasetServiceDAO; + + @Mock private StudyDAO studyDAO; + + @BeforeEach + void initService() { + service = + new OpenSearchService( + esClient, + esConfig, + dacDAO, + userDao, + ontologyService, + institutionDAO, + datasetDAO, + datasetServiceDAO, + studyDAO); + } + + private void mockElasticSearchResponse(String body) throws IOException { + Response response = mock(Response.class); + String reasonPhrase = fromStatusCode(200).getReasonPhrase(); + StatusLine status = new StatusLine(new BasicHttpResponse(200, reasonPhrase)); + HttpEntity entity = new StringEntity(body, ContentType.APPLICATION_JSON); + + when(esClient.performRequest(any())).thenReturn(response); + when(response.getStatusLine()).thenReturn(status); + when(response.getEntity()).thenReturn(entity); + } + + private Institution createInstitution() { + Institution institution = new Institution(); + institution.setId(randomInt(1, 1000)); + return institution; + } + + private User createUser(int start, int max) { + User user = new User(); + user.setUserId(randomInt(start, max)); + user.setDisplayName(randomAlphabetic(10)); + user.setEmail(randomAlphabetic(10)); + Institution i = createInstitution(); + user.setInstitution(i); + user.setInstitutionId(i.getId()); + return user; + } + + private Dataset createDataset(User user, User updateUser, DataUse dataUse, Dac dac) { + Dataset dataset = new Dataset(); + dataset.setDatasetId(randomInt(1, 100)); + dataset.setAlias(dataset.getDatasetId()); + dataset.setDatasetIdentifier(); + dataset.setDeletable(true); + dataset.setName(randomAlphabetic(10)); + dataset.setDatasetName(dataset.getName()); + dataset.setDacId(dac.getDacId()); + dataset.setDacApproval(true); + dataset.setDataUse(dataUse); + dataset.setCreateUser(user); + dataset.setUpdateUserId(updateUser.getUserId()); + dataset.setCreateUserId(user.getUserId()); + dataset.setNihInstitutionalCertificationFile(new FileStorageObject()); + return dataset; + } + + private Dac createDac() { + Dac dac = new Dac(); + dac.setDacId(randomInt(1, 100)); + dac.setName(randomAlphabetic(10)); + return dac; + } + + private Study createStudy(User user) { + Study study = new Study(); + study.setName(randomAlphabetic(10)); + study.setDescription(randomAlphabetic(20)); + study.setStudyId(randomInt(1, 100)); + study.setPiName(randomAlphabetic(10)); + study.setDataTypes(List.of(randomAlphabetic(10))); + study.setPublicVisibility(true); + study.setCreateUserEmail(user.getEmail()); + study.setCreateUserId(user.getUserId()); + study.setCreateUserEmail(user.getEmail()); + return study; + } + + private StudyProperty createStudyProperty(String key, PropertyType type) { + StudyProperty prop = new StudyProperty(); + prop.setKey(key); + prop.setType(type); + switch (type) { + case Boolean -> prop.setValue(true); + case Json -> { + var val = new JsonArray(); + val.add(randomAlphabetic(10)); + prop.setValue(val); + } + case Number -> prop.setValue(randomInt(1, 100)); + default -> prop.setValue(randomAlphabetic(10)); + } + return prop; + } + + private DatasetProperty createDatasetProperty( + String schemaProp, PropertyType type, String propertyName) { + DatasetProperty prop = new DatasetProperty(); + prop.setSchemaProperty(schemaProp); + prop.setPropertyType(type); + prop.setPropertyName(propertyName); + switch (type) { + case Boolean -> prop.setPropertyValue(true); + case Number -> prop.setPropertyValue(randomInt(1, 100)); + default -> prop.setPropertyValue(randomAlphabetic(10)); + } + return prop; + } + + private DataUseSummary createDataUseSummary() { + DataUseSummary dataUseSummary = new DataUseSummary(); + dataUseSummary.setPrimary(List.of(new DataUseTerm("DS", "Description"))); + dataUseSummary.setPrimary(List.of(new DataUseTerm("NMDS", "Description"))); + return dataUseSummary; + } + + /** Private container record to consolidate dataset and associated object creation */ + private record DatasetRecord( + User createUser, User updateUser, Dac dac, Dataset dataset, Study study) {} + + private DatasetRecord createDatasetRecord() { + User user = createUser(1, 100); + User updateUser = createUser(101, 200); + Dac dac = createDac(); + Study study = createStudy(user); + study.addProperties( + createStudyProperty("dbGaPPhsID", PropertyType.String), + createStudyProperty("phenotypeIndication", PropertyType.String), + createStudyProperty("species", PropertyType.String), + createStudyProperty("dataCustodianEmail", PropertyType.Json)); + Dataset dataset = createDataset(user, updateUser, new DataUse(), dac); + dataset.setProperties( + Set.of( + createDatasetProperty("accessManagement", PropertyType.Boolean, "accessManagement"), + createDatasetProperty("numberOfParticipants", PropertyType.Number, "# of participants"), + createDatasetProperty("url", PropertyType.String, "url"), + createDatasetProperty("dataLocation", PropertyType.String, "dataLocation"))); + dataset.setStudy(study); + return new DatasetRecord(user, updateUser, dac, dataset, study); + } + + @Test + void testAsyncESIndexUpdate() { + DatasetRecord datasetRecord = createDatasetRecord(); + datasetRecord.dataset.setDataUse(new DataUseBuilder().setGeneralUse(true).build()); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + when(datasetDAO.findDatasetById(datasetRecord.dataset.getDatasetId())) + .thenReturn(datasetRecord.dataset); + OpenSearchService elasticSearchSpy = spy(service); + // Call the async method ... + elasticSearchSpy.asyncDatasetInESIndex( + datasetRecord.dataset.getDatasetId(), datasetRecord.createUser, true); + // Ensure that the synchronous method was called with the expected parameters + verify(elasticSearchSpy, timeout(1000)) + .synchronizeDatasetInESIndex( + datasetRecord.dataset, datasetRecord.dataset.getCreateUser(), true); + } + + @Test + void testToDatasetTerm_UserInfo() { + DatasetRecord datasetRecord = createDatasetRecord(); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + when(userDao.findUserById(datasetRecord.updateUser.getUserId())) + .thenReturn(datasetRecord.updateUser); + when(institutionDAO.findInstitutionById(datasetRecord.createUser.getInstitutionId())) + .thenReturn(datasetRecord.createUser.getInstitution()); + when(institutionDAO.findInstitutionById(datasetRecord.updateUser.getInstitutionId())) + .thenReturn(datasetRecord.updateUser.getInstitution()); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + assertEquals(datasetRecord.createUser.getUserId(), term.getCreateUserId()); + assertEquals(datasetRecord.createUser.getDisplayName(), term.getCreateUserDisplayName()); + assertEquals(datasetRecord.createUser.getUserId(), term.getSubmitter().userId()); + assertEquals(datasetRecord.createUser.getDisplayName(), term.getSubmitter().displayName()); + assertEquals( + datasetRecord.createUser.getInstitutionId(), term.getSubmitter().institution().id()); + assertEquals( + datasetRecord.createUser.getInstitution().getName(), + term.getSubmitter().institution().name()); + assertEquals(datasetRecord.updateUser.getUserId(), term.getUpdateUser().userId()); + assertEquals(datasetRecord.updateUser.getDisplayName(), term.getUpdateUser().displayName()); + assertEquals( + datasetRecord.updateUser.getInstitutionId(), term.getUpdateUser().institution().id()); + assertEquals( + datasetRecord.updateUser.getInstitution().getName(), + term.getUpdateUser().institution().name()); + } + + @Test + void testToDatasetTerm_StudyInfo() { + DatasetRecord datasetRecord = createDatasetRecord(); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + when(userDao.findUserById(datasetRecord.updateUser.getUserId())) + .thenReturn(datasetRecord.updateUser); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + assertEquals(datasetRecord.study.getDescription(), term.getStudy().getDescription()); + assertEquals(datasetRecord.study.getName(), term.getStudy().getStudyName()); + assertEquals(datasetRecord.study.getStudyId(), term.getStudy().getStudyId()); + Optional phsIdProp = + datasetRecord.study.getProperties().stream() + .filter(p -> p.getKey().equals("dbGaPPhsID")) + .findFirst(); + assertTrue(phsIdProp.isPresent()); + assertEquals(phsIdProp.get().getValue().toString(), term.getStudy().getPhsId()); + Optional phenoProp = + datasetRecord.study.getProperties().stream() + .filter(p -> p.getKey().equals("phenotypeIndication")) + .findFirst(); + assertTrue(phenoProp.isPresent()); + assertEquals(phenoProp.get().getValue().toString(), term.getStudy().getPhenotype()); + Optional speciesProp = + datasetRecord.study.getProperties().stream() + .filter(p -> p.getKey().equals("species")) + .findFirst(); + assertTrue(speciesProp.isPresent()); + assertEquals(speciesProp.get().getValue().toString(), term.getStudy().getSpecies()); + assertEquals(datasetRecord.study.getPiName(), term.getStudy().getPiName()); + assertEquals(datasetRecord.study.getCreateUserEmail(), term.getStudy().getDataSubmitterEmail()); + assertEquals(datasetRecord.study.getCreateUserId(), term.getStudy().getDataSubmitterId()); + Optional custodianProp = + datasetRecord.study.getProperties().stream() + .filter(p -> p.getKey().equals("dataCustodianEmail")) + .findFirst(); + assertTrue(custodianProp.isPresent()); + String termCustodians = + GsonUtil.getInstance().toJson(term.getStudy().getDataCustodianEmail(), ArrayList.class); + assertEquals(custodianProp.get().getValue().toString(), termCustodians); + assertEquals(datasetRecord.study.getPublicVisibility(), term.getStudy().getPublicVisibility()); + assertEquals(datasetRecord.study.getDataTypes(), term.getStudy().getDataTypes()); + } + + @Test + void testToDatasetTerm_StudyAssets() { + DatasetRecord datasetRecord = createDatasetRecord(); + Map assetsMap = Map.of("key", List.of("value1", "value2")); + String assetsJson = GsonUtil.getInstance().toJson(assetsMap); + StudyProperty assetsProp = new StudyProperty(); + assetsProp.setStudyId(datasetRecord.study.getStudyId()); + assetsProp.setKey("assets"); + assetsProp.setType(PropertyType.Json); + assetsProp.setValue(GsonUtil.getInstance().fromJson(assetsJson, Object.class)); + datasetRecord.study.addProperty(assetsProp); + + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + when(userDao.findUserById(datasetRecord.updateUser.getUserId())) + .thenReturn(datasetRecord.updateUser); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + + assertEquals(assetsMap, term.getStudy().getAssets()); + } + + @Test + void testToDatasetTerm_DatasetInfo() { + DataAccessRequest dar1 = new DataAccessRequest(); + dar1.setUserId(1); + DataAccessRequest dar2 = new DataAccessRequest(); + dar2.setUserId(2); + DataUseSummary dataUseSummary = createDataUseSummary(); + DatasetRecord datasetRecord = createDatasetRecord(); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + when(userDao.findUserById(datasetRecord.updateUser.getUserId())) + .thenReturn(datasetRecord.updateUser); + LibraryCard card1 = new LibraryCard(); + card1.setUserId(dar1.getUserId()); + LibraryCard card2 = new LibraryCard(); + card2.setUserId(dar2.getUserId()); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + when(ontologyService.translateDataUseSummary(any())).thenReturn(dataUseSummary); + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + + assertEquals(datasetRecord.dataset.getDatasetId(), term.getDatasetId()); + assertEquals(datasetRecord.dataset.getDatasetIdentifier(), term.getDatasetIdentifier()); + assertEquals(datasetRecord.dataset.getDeletable(), term.getDeletable()); + assertEquals(datasetRecord.dataset.getName(), term.getDatasetName()); + assertEquals(datasetRecord.dataset.getDatasetName(), term.getDatasetName()); + + Optional countProp = + datasetRecord.dataset.getProperties().stream() + .filter(p -> p.getSchemaProperty().equals("numberOfParticipants")) + .findFirst(); + assertTrue(countProp.isPresent()); + assertEquals( + Integer.valueOf(countProp.get().getPropertyValue().toString()), term.getParticipantCount()); + assertEquals(dataUseSummary, term.getDataUse()); + Optional locationProp = + datasetRecord.dataset.getProperties().stream() + .filter(p -> p.getSchemaProperty().equals("dataLocation")) + .findFirst(); + assertTrue(locationProp.isPresent()); + assertEquals(locationProp.get().getPropertyValue().toString(), term.getDataLocation()); + Optional urlProp = + datasetRecord.dataset.getProperties().stream() + .filter(p -> p.getSchemaProperty().equals("url")) + .findFirst(); + assertTrue(urlProp.isPresent()); + assertEquals(urlProp.get().getPropertyValue().toString(), term.getUrl()); + assertEquals(datasetRecord.dataset.getDacApproval(), term.getDacApproval()); + Optional accessManagementProp = + datasetRecord.dataset.getProperties().stream() + .filter(p -> p.getSchemaProperty().equals("accessManagement")) + .findFirst(); + assertTrue(accessManagementProp.isPresent()); + assertEquals( + accessManagementProp.get().getPropertyValue().toString(), term.getAccessManagement()); + } + + @Test + void testToDatasetTerm_DacInfo() { + DatasetRecord datasetRecord = createDatasetRecord(); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + + assertEquals(datasetRecord.dataset.getDacApproval(), term.getDacApproval()); + assertEquals(datasetRecord.dac.getDacId(), term.getDacId()); + assertEquals(datasetRecord.dac.getDacId(), term.getDac().dacId()); + assertEquals(datasetRecord.dac.getName(), term.getDac().dacName()); + } + + @Test + void testToDatasetTerm_NIHInstitutionalCertification() { + DatasetRecord datasetRecord = createDatasetRecord(); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + assertEquals( + datasetRecord.dataset.getNihInstitutionalCertificationFile() != null, + term.getHasInstitutionCertification()); + } + + @Test + void testToDatasetTerm_Missing_NIHInstitutionalCertification() { + DatasetRecord datasetRecord = createDatasetRecord(); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + datasetRecord.dataset.setNihInstitutionalCertificationFile(null); + DatasetTerm term = service.toDatasetTerm(datasetRecord.dataset); + assertEquals(null, term.getHasInstitutionCertification()); + } + + @Test + void testToDatasetTerm_StringNumberOfParticipants() { + User user = createUser(1, 100); + User updateUser = createUser(101, 200); + Dac dac = createDac(); + Study study = createStudy(user); + study.addProperties( + createStudyProperty("phenotypeIndication", PropertyType.String), + createStudyProperty("species", PropertyType.String), + createStudyProperty("dataCustodianEmail", PropertyType.Json)); + Dataset dataset = createDataset(user, updateUser, new DataUse(), dac); + dataset.setProperties( + Set.of( + createDatasetProperty("numberOfParticipants", PropertyType.String, "# of participants"), + createDatasetProperty("url", PropertyType.String, "url"))); + dataset.setStudy(study); + DatasetRecord datasetRecord = new DatasetRecord(user, updateUser, dac, dataset, study); + when(dacDAO.findById(any())).thenReturn(dac); + when(userDao.findUserById(user.getUserId())).thenReturn(user); + when(dacDAO.findById(any())).thenReturn(datasetRecord.dac); + when(userDao.findUserById(datasetRecord.createUser.getUserId())) + .thenReturn(datasetRecord.createUser); + assertDoesNotThrow(() -> service.toDatasetTerm(dataset)); + } + + @Test + void testToDatasetTermIncomplete() { + Dataset dataset = new Dataset(); + dataset.setDatasetId(100); + dataset.setAlias(10); + dataset.setDatasetIdentifier(); + dataset.setProperties(Set.of()); + + DatasetTerm term = service.toDatasetTerm(dataset); + + assertEquals(dataset.getDatasetId(), term.getDatasetId()); + assertEquals(dataset.getDatasetIdentifier(), term.getDatasetIdentifier()); + } + + @Test + void testToDatasetTermNullDatasetProps() { + Dataset dataset = new Dataset(); + assertDoesNotThrow(() -> service.toDatasetTerm(dataset)); + } + + @Test + void testToDatasetTermNullStudyProps() { + Dataset dataset = new Dataset(); + Study study = new Study(); + study.setName(randomAlphabetic(10)); + study.setDescription(randomAlphabetic(20)); + study.setStudyId(randomInt(1, 100)); + dataset.setStudy(study); + assertDoesNotThrow(() -> service.toDatasetTerm(dataset)); + } + + @Captor ArgumentCaptor request; + + @Test + void testIndexDatasetTerms() throws IOException { + DatasetTerm term1 = new DatasetTerm(); + term1.setDatasetId(1); + DatasetTerm term2 = new DatasetTerm(); + term2.setDatasetId(2); + User user = new User(); + user.setUserId(1); + String datasetIndexName = randomAlphabetic(10); + + when(esConfig.getDatasetIndexName()).thenReturn(datasetIndexName); + mockElasticSearchResponse(""); + + try (var response = service.indexDatasetTerms(List.of(term1, term2), user)) { + verify(esClient).performRequest(request.capture()); + Request capturedRequest = request.getValue(); + assertEquals("PUT", capturedRequest.getMethod()); + assertEquals( + """ + { "index": {"_index": "dataset", "_id": "1"} } + {"datasetId":1} + { "index": {"_index": "dataset", "_id": "2"} } + {"datasetId":2} + + """, + new String( + capturedRequest.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void testIndexDatasets() throws Exception { + DatasetRecord datasetRecord = createDatasetRecord(); + Dataset dataset1 = datasetRecord.dataset; + Dataset dataset2 = + createDataset( + datasetRecord.createUser, + datasetRecord.updateUser, + new DataUseBuilder().setGeneralUse(true).build(), + datasetRecord.dac); + dataset2.setStudy(datasetRecord.study); + when(datasetDAO.findDatasetById(dataset1.getDatasetId())).thenReturn(dataset1); + when(datasetDAO.findDatasetById(dataset2.getDatasetId())).thenReturn(dataset2); + when(userDao.findUserById(dataset1.getCreateUserId())).thenReturn(datasetRecord.createUser); + when(userDao.findUserById(dataset2.getCreateUserId())).thenReturn(datasetRecord.createUser); + org.opensearch.client.Response mockResponse = mock(); + when(esClient.performRequest(any())).thenReturn(mockResponse); + when(mockResponse.getEntity()).thenReturn(new StringEntity("response body")); + StatusLine statusLine = mock(); + when(mockResponse.getStatusLine()).thenReturn(statusLine); + when(statusLine.getStatusCode()).thenReturn(200); + + try (var ignored = + service.indexDatasets( + List.of(dataset1.getDatasetId(), dataset2.getDatasetId()), datasetRecord.createUser)) { + // Each dataset should be looked up once when defining the term and a second time + // when updating the indexed date. + verify(datasetDAO, times(2)).findDatasetById(dataset1.getDatasetId()); + verify(datasetDAO, times(2)).findDatasetById(dataset2.getDatasetId()); + } + } + + @Test + void testSearchDatasets() throws IOException { + String query = "{ \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }"; + + /* + * FIXME: this approach is kind of hacky, we stick both the validation response and the search + * response in the same body, and then rely on Gson to parse these into separate objects. + * Ideally each request and response should be mocked separately, but this would involve many + * more classes and methods. Alternately, it is possible to just mock the Gson parsing, but + * this seems to affect the results of the other tests. + */ + mockElasticSearchResponse("{\"valid\":true,\"hits\":{\"hits\":[]}}"); + + try (var response = service.searchDatasets(query)) { + assertEquals(200, response.getStatus()); + } + } + + @Test + void testSearchDatasetsStream() throws IOException { + String query = "{ \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }"; + String mockResponse = "{\"valid\":true,\"hits\":{\"hits\":[]}}"; + mockElasticSearchResponse(mockResponse); + try (var inputStream = service.searchDatasetsStream(query)) { + String received = IOUtils.toString(inputStream, Charset.defaultCharset()); + assertEquals(mockResponse, received); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "{ \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }", + "{ \"from\": 0, \"size\": 100, \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }", + "{ \"sort\": [\"datasetIdentifier\"], \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }", + "{ \"from\": 0, \"size\": 100, \"sort\": [\"datasetIdentifier\"], \"query\": { \"query_string\": { \"query\": \"(GRU) AND (HMB)\" } } }", + }) + void testValidateQuerySuccess(String query) throws IOException { + mockElasticSearchResponse("{\"valid\":true}"); + assertTrue(service.validateQuery(query)); + } + + @Test + void testValidateQueryEmpty() throws IOException { + String query = "{}"; + + Response response = mock(Response.class); + String reasonPhrase = fromStatusCode(400).getReasonPhrase(); + StatusLine status = new StatusLine(new BasicHttpResponse(400, reasonPhrase)); + when(esClient.performRequest(any())).thenReturn(response); + when(response.getStatusLine()).thenReturn(status); + + assertThrows(IOException.class, () -> service.validateQuery(query)); + } + + @Test + void testValidateQueryInvalid() throws IOException { + String query = "{ \"bad\": [\"and\", \"invalid\"] }"; + + mockElasticSearchResponse("{\"valid\":false}"); + + assertFalse(service.validateQuery(query)); + } + + @Test + void testIndexDatasetIds() throws Exception { + User user = new User(); + user.setUserId(1); + Gson gson = GsonUtil.buildGson(); + Dataset dataset = new Dataset(); + dataset.setDatasetId(randomInt(10, 100)); + String esResponseBody = + """ + { + "took": 2, + "errors": false, + "items": [ + { + "index": { + "_index": "dataset", + "_type": "dataset", + "_id": "%d", + "_version": 3, + "result": "updated", + "_shards": { + "total": 2, + "successful": 1, + "failed": 0 + }, + "created": false, + "status": 200 + } + } + ] + } + """; + + when(datasetDAO.findDatasetById(dataset.getDatasetId())).thenReturn(dataset); + mockESClientResponse(200, esResponseBody.formatted(dataset.getDatasetId())); + StreamingOutput output = service.indexDatasetIds(List.of(dataset.getDatasetId()), user); + var baos = new ByteArrayOutputStream(); + output.write(baos); + var entityString = baos.toString(); + Type listOfEsResponses = new TypeToken>() {}.getType(); + List responseList = gson.fromJson(entityString, listOfEsResponses); + assertEquals(1, responseList.size()); + JsonArray items = responseList.get(0).getAsJsonArray("items"); + assertEquals(1, items.size()); + assertEquals( + dataset.getDatasetId(), + items.get(0).getAsJsonObject().getAsJsonObject("index").get("_id").getAsInt()); + } + + @Test + void testIndexDatasetIdsErrors() throws Exception { + User user = new User(); + user.setUserId(1); + Gson gson = GsonUtil.buildGson(); + Dataset dataset = new Dataset(); + dataset.setDatasetId(randomInt(10, 100)); + when(datasetDAO.findDatasetById(dataset.getDatasetId())).thenReturn(dataset); + mockESClientResponse(500, "error condition"); + + StreamingOutput output = service.indexDatasetIds(List.of(dataset.getDatasetId()), user); + var baos = new ByteArrayOutputStream(); + output.write(baos); + JsonArray jsonArray = gson.fromJson(baos.toString(), JsonArray.class); + assertEquals(0, jsonArray.size()); + } + + // Helper method to mock an OpenSearch Client response + private void mockESClientResponse(int status, String body) throws Exception { + var esClientResponse = mock(org.opensearch.client.Response.class); + var statusLine = mock(StatusLine.class); + when(esClientResponse.getStatusLine()).thenReturn(statusLine); + when(statusLine.getStatusCode()).thenReturn(status); + var httpEntity = mock(HttpEntity.class); + if (status == 200) { + when(httpEntity.getContent()) + .thenReturn(new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8))); + when(esClientResponse.getEntity()).thenReturn(httpEntity); + } + when(esClient.performRequest(any())).thenReturn(esClientResponse); + } + + @Test + void testIndexStudyWithDatasets() { + User user = new User(); + user.setUserId(1); + Study study = new Study(); + study.setStudyId(1); + Dataset d = new Dataset(); + d.setDatasetId(1); + study.addDatasetId(d.getDatasetId()); + when(studyDAO.findStudyById(any())).thenReturn(study); + + assertDoesNotThrow(() -> service.indexStudy(1, user)); + } + + @Test + void testIndexStudyWithNoDatasets() { + User user = new User(); + user.setUserId(1); + Study study = new Study(); + study.setStudyId(1); + when(studyDAO.findStudyById(any())).thenReturn(study); + + assertDoesNotThrow( + () -> { + try (var response = service.indexStudy(1, user)) { + assertEquals(HttpStatusCodes.STATUS_CODE_NOT_FOUND, response.getStatus()); + } + }); + } + + @Test + void testUpdateDatasetIndexDateWithValue() { + assertDoesNotThrow(() -> service.updateDatasetIndexDate(1, 1, Instant.now())); + } + + @Test + void testDeleteDatasetIndexWhenDatasetExists() throws Exception { + Dataset dataset = new Dataset(); + when(datasetDAO.findDatasetById(any())).thenReturn(dataset); + assertDoesNotThrow(() -> service.updateDatasetIndexDate(1, 1, null)); + verify(datasetServiceDAO).updateDatasetIndex(any(), any(), any()); + } + + @Test + void testDeleteDatasetIndexWhenDatasetIsNull() throws Exception { + when(datasetDAO.findDatasetById(any())).thenReturn(null); + assertDoesNotThrow(() -> service.updateDatasetIndexDate(1, 1, null)); + verify(datasetServiceDAO, never()).updateDatasetIndex(any(), any(), any()); + } + + @Test + void testInvalidResultWindow_ValidQueryWithinLimits() { + String query = "{\"size\": 100, \"from\": 0}"; + assertFalse(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_DoubleValuesThrowClassCastException() { + String query = "{\"size\": 100.0, \"from\": 0.0}"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_StringValuesThrowClassCastException() { + String query = "{\"size\": \"100\", \"from\": \"0\"}"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_ValidQueryAtLimit() { + String query = "{\"size\": 5000, \"from\": 5000}"; + assertFalse(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_ExceedsMaxResultWindow() { + String query = "{\"size\": 5000, \"from\": 5001}"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_ExceedsMaxResultWindowLargeValues() { + String query = "{\"size\": 10000, \"from\": 1}"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_DefaultValues() { + String query = "{}"; + assertFalse(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_OnlySizeProvided() { + String query = "{\"size\": 50}"; + assertFalse(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_OnlyFromProvided() { + String query = "{\"from\": 100}"; + assertFalse(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_InvalidJsonFormat() { + String query = "{invalid json}"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_NullQuery() { + assertTrue(service.invalidResultWindow(null)); + } + + @Test + void testInvalidResultWindow_EmptyString() { + String query = ""; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_NonJsonString() { + String query = "not a json string"; + assertTrue(service.invalidResultWindow(query)); + } + + @Test + void testInvalidResultWindow_ExtraFieldsInQuery() { + String query = "{\"size\": 100, \"from\": 50, \"query\": {\"match_all\": {}}}"; + assertFalse(service.invalidResultWindow(query)); + } +} diff --git a/src/test/java/org/broadinstitute/consent/http/service/VoteServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/VoteServiceTest.java index b14c35e8e8..8de8a0c8cd 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/VoteServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/VoteServiceTest.java @@ -78,6 +78,7 @@ class VoteServiceTest extends AbstractTestHelper { @Mock private ElectionDAO electionDAO; @Mock private EmailService emailService; @Mock private ElasticSearchService elasticSearchService; + @Mock private OpenSearchService openSearchService; @Mock private UseRestrictionConverter useRestrictionConverter; @Mock private VoteDAO voteDAO; @Mock private VoteServiceDAO voteServiceDAO; @@ -94,6 +95,7 @@ void initService() { electionDAO, emailService, elasticSearchService, + openSearchService, useRestrictionConverter, voteDAO, voteServiceDAO); diff --git a/src/test/resources/consent-config.yml b/src/test/resources/consent-config.yml index 0f7061a067..137e0c3490 100644 --- a/src/test/resources/consent-config.yml +++ b/src/test/resources/consent-config.yml @@ -56,5 +56,11 @@ elasticSearch: - localhost indexName: iName datasetIndexName: dataset +openSearch: + servers: + - localhost + indexName: iName + datasetIndexName: dataset + port: 9201 oidcConfiguration: authorityEndpoint: http://localhost:8000