Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,18 @@
<version>9.2.1</version>
</dependency>

<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-client</artifactId>
<version>3.3.2</version>
</dependency>

<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-java</artifactId>
<version>3.3.0</version>
</dependency>

<dependency>
<groupId>org.webjars</groupId>
<artifactId>swagger-ui</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.broadinstitute.consent.http.health.ElasticSearchHealthCheck;
import org.broadinstitute.consent.http.health.GCSHealthCheck;
import org.broadinstitute.consent.http.health.OntologyHealthCheck;
import org.broadinstitute.consent.http.health.OpenSearchHealthCheck;
import org.broadinstitute.consent.http.health.SamHealthCheck;
import org.broadinstitute.consent.http.health.SendGridHealthCheck;
import org.broadinstitute.consent.http.models.AuthUser;
Expand Down Expand Up @@ -96,6 +97,7 @@ public class ConsentApplication extends Application<ConsentConfiguration> {

public static final String GCS_CHECK = "google-cloud-storage";
public static final String ES_CHECK = "elastic-search";
public static final String OS_CHECK = "opensearch";
public static final String ONTOLOGY_CHECK = "ontology";
public static final String SAM_CHECK = "sam";
public static final String SG_CHECK = "sendgrid";
Expand Down Expand Up @@ -146,6 +148,7 @@ public void run(ConsentConfiguration config, Environment env) {
// Health Checks
env.healthChecks().register(GCS_CHECK, injector.getInstance(GCSHealthCheck.class));
env.healthChecks().register(ES_CHECK, injector.getInstance(ElasticSearchHealthCheck.class));
env.healthChecks().register(OS_CHECK, injector.getInstance(OpenSearchHealthCheck.class));
env.healthChecks().register(ONTOLOGY_CHECK, injector.getInstance(OntologyHealthCheck.class));
env.healthChecks().register(SAM_CHECK, injector.getInstance(SamHealthCheck.class));
env.healthChecks().register(SG_CHECK, injector.getInstance(SendGridHealthCheck.class));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration;
import org.broadinstitute.consent.http.configurations.GoogleOAuth2Config;
import org.broadinstitute.consent.http.configurations.MailConfiguration;
import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration;
import org.broadinstitute.consent.http.configurations.ServicesConfiguration;
import org.broadinstitute.consent.http.db.AcknowledgementDAO;
import org.broadinstitute.consent.http.db.CounterDAO;
Expand Down Expand Up @@ -65,6 +66,7 @@
import org.broadinstitute.consent.http.service.NihService;
import org.broadinstitute.consent.http.service.OidcService;
import org.broadinstitute.consent.http.service.OntologyService;
import org.broadinstitute.consent.http.service.OpenSearchService;
import org.broadinstitute.consent.http.service.SupportRequestService;
import org.broadinstitute.consent.http.service.UseRestrictionConverter;
import org.broadinstitute.consent.http.service.UserService;
Expand All @@ -81,6 +83,7 @@
import org.broadinstitute.consent.http.service.dao.VoteServiceDAO;
import org.broadinstitute.consent.http.service.feature.InstitutionAndLibraryCardEnforcement;
import org.broadinstitute.consent.http.service.ontology.ElasticSearchSupport;
import org.broadinstitute.consent.http.service.ontology.OpenSearchSupport;
import org.broadinstitute.consent.http.service.sam.SamService;
import org.broadinstitute.consent.http.util.HttpClientUtil;
import org.broadinstitute.consent.http.util.gson.GsonUtil;
Expand Down Expand Up @@ -208,6 +211,11 @@ ElasticSearchConfiguration providesElasticSearchConfiguration() {
return config.getElasticSearchConfiguration();
}

@Provides
OpenSearchConfiguration providesOpenSearchConfiguration() {
return config.getOpenSearchConfiguration();
}

@Provides
MailConfiguration providesMailConfiguration() {
return config.getMailConfiguration();
Expand Down Expand Up @@ -323,6 +331,7 @@ DatasetService providesDatasetService() {
providesDaaDAO(),
providesDacDAO(),
providesElasticSearchService(),
providesOpenSearchService(),
providesEmailService(),
providesOntologyService(),
providesStudyDAO(),
Expand Down Expand Up @@ -412,6 +421,7 @@ VoteService providesVoteService() {
providesElectionDAO(),
providesEmailService(),
providesElasticSearchService(),
providesOpenSearchService(),
providesUseRestrictionConverter(),
providesVoteDAO(),
providesVoteServiceDAO());
Expand Down Expand Up @@ -487,6 +497,20 @@ ElasticSearchService providesElasticSearchService() {
providesStudyDAO());
}

@Provides
OpenSearchService providesOpenSearchService() {
return new OpenSearchService(
OpenSearchSupport.createRestClient(config.getOpenSearchConfiguration()),
config.getOpenSearchConfiguration(),
providesDacDAO(),
providesUserDAO(),
providesOntologyService(),
providesInstitutionDAO(),
providesDatasetDAO(),
providesDatasetServiceDAO(),
providesStudyDAO());
}

@Provides
UserDAO providesUserDAO() {
return userDAO;
Expand Down Expand Up @@ -584,6 +608,7 @@ DatasetRegistrationService providesDatasetRegistrationService() {
providesDatasetServiceDAO(),
providesGCSService(),
providesElasticSearchService(),
providesOpenSearchService(),
providesStudyDAO(),
providesEmailService());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public JerseyClientConfiguration getJerseyClientConfiguration() {
@Valid @NotNull @JsonProperty
private final ElasticSearchConfiguration elasticSearch = new ElasticSearchConfiguration();

@Valid @NotNull @JsonProperty
private final OpenSearchConfiguration openSearch = new OpenSearchConfiguration();

@Valid @NotNull @JsonProperty
private final OidcConfiguration oidcConfiguration = new OidcConfiguration();

Expand Down Expand Up @@ -73,6 +76,10 @@ public ElasticSearchConfiguration getElasticSearchConfiguration() {
return elasticSearch;
}

public OpenSearchConfiguration getOpenSearchConfiguration() {
return openSearch;
}

public OidcConfiguration getOidcConfiguration() {
return oidcConfiguration;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package org.broadinstitute.consent.http.configurations;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import jakarta.validation.constraints.NotNull;
import java.util.List;

@JsonIgnoreProperties(ignoreUnknown = true)
public class OpenSearchConfiguration {

@NotNull private String indexName;

@NotNull private List<String> servers;

@NotNull private String datasetIndexName;
Comment on lines +10 to +14
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are staging the OpenSearch rollout over time and we don't have values for production, this will likely cause the app to fail on startup.


/** This is configurable for testing purposes */
private int port = 9201;

public List<String> getServers() {
return servers;
}

public void setServers(List<String> servers) {
this.servers = servers;
}

public String getIndexName() {
return indexName;
}

public void setIndexName(String indexName) {
this.indexName = indexName;
}

public int getPort() {
return port;
}

public void setPort(int port) {
this.port = port;
}

public String getDatasetIndexName() {
return datasetIndexName;
}

public void setDatasetIndexName(String datasetIndexName) {
this.datasetIndexName = datasetIndexName;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.broadinstitute.consent.http.health;

import static org.broadinstitute.consent.http.service.ontology.OpenSearchSupport.jsonHeader;

import com.codahale.metrics.health.HealthCheck;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import io.dropwizard.lifecycle.Managed;
import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.io.IOUtils;
import org.broadinstitute.consent.http.configurations.OpenSearchConfiguration;
import org.broadinstitute.consent.http.service.ontology.OpenSearchSupport;
import org.opensearch.client.Request;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.Response;
import org.opensearch.client.RestClient;

public class OpenSearchHealthCheck extends HealthCheck implements Managed {

private final RestClient client;

@Override
public void start() throws Exception {}

Check failure on line 25 in src/main/java/org/broadinstitute/consent/http/health/OpenSearchHealthCheck.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a nested comment explaining why this method is empty, throw an UnsupportedOperationException or complete the implementation.

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozuHGmrRvUMRnyBJ&open=AZsDozuHGmrRvUMRnyBJ&pullRequest=2765

@Override
public void stop() throws Exception {
if (client != null) {
client.close();
}
}

@Inject
public OpenSearchHealthCheck(OpenSearchConfiguration config) {
this.client = OpenSearchSupport.createRestClient(config);
}

@Override
protected Result check() throws Exception {
try {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
builder.addHeader(jsonHeader.getName(), jsonHeader.getValue());
Request request = new Request("GET", OpenSearchSupport.getClusterHealthPath());
request.setOptions(builder.build());
Response esResponse = client.performRequest(request);
if (esResponse.getStatusLine().getStatusCode() != 200) {
return Result.unhealthy(
"Invalid health check request: " + esResponse.getStatusLine().getReasonPhrase());
}
String stringResponse =
IOUtils.toString(esResponse.getEntity().getContent(), Charset.defaultCharset());
JsonObject jsonResponse = JsonParser.parseString(stringResponse).getAsJsonObject();
String status = jsonResponse.get("status").getAsString();
if (status.equalsIgnoreCase("red")) {
return Result.unhealthy("ClusterHealth is RED\n" + jsonResponse);
}
if (status.equalsIgnoreCase("yellow")) {
return Result.healthy("ClusterHealth is YELLOW\n" + jsonResponse);
}
} catch (IOException e) {
return Result.unhealthy("Unable to connect to OpenSearch");
}
return Result.healthy("ClusterHealth is GREEN");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import org.broadinstitute.consent.http.service.DatasetRegistrationService;
import org.broadinstitute.consent.http.service.DatasetService;
import org.broadinstitute.consent.http.service.ElasticSearchService;
import org.broadinstitute.consent.http.service.OpenSearchService;
import org.broadinstitute.consent.http.service.TDRService;
import org.broadinstitute.consent.http.service.UserService;
import org.broadinstitute.consent.http.util.JsonSchemaUtil;
Expand All @@ -72,6 +73,7 @@
private final TDRService tdrService;
private final UserService userService;
private final ElasticSearchService elasticSearchService;
private final OpenSearchService openSearchService;

private final JsonSchemaUtil jsonSchemaUtil;
private final GCSService gcsService;
Expand All @@ -82,13 +84,15 @@
UserService userService,
DatasetRegistrationService datasetRegistrationService,
ElasticSearchService elasticSearchService,
OpenSearchService openSearchService,
TDRService tdrService,
GCSService gcsService) {
this.datasetService = datasetService;
this.userService = userService;
this.datasetRegistrationService = datasetRegistrationService;
this.gcsService = gcsService;
this.elasticSearchService = elasticSearchService;
this.openSearchService = openSearchService;
this.tdrService = tdrService;
this.jsonSchemaUtil = new JsonSchemaUtil();
}
Expand Down Expand Up @@ -222,6 +226,7 @@
}
Dataset patched = datasetRegistrationService.patchDataset(datasetId, user, patch);
elasticSearchService.synchronizeDatasetInESIndex(patched, user, false);
openSearchService.synchronizeDatasetInESIndex(patched, user, false);
return Response.ok(patched).build();
} catch (Exception e) {
return createExceptionResponse(e);
Expand Down Expand Up @@ -376,7 +381,12 @@
}
try (var deleteResponse = elasticSearchService.deleteIndex(datasetId, user.getUserId())) {
if (!HttpStatusCodes.isSuccess(deleteResponse.getStatus())) {
logWarn("Unable to delete index for dataset: " + datasetId);
logWarn("Unable to delete Elasticsearch index for dataset: " + datasetId);
}
}
try (var deleteResponse = openSearchService.deleteIndex(datasetId, user.getUserId())) {
if (!HttpStatusCodes.isSuccess(deleteResponse.getStatus())) {
logWarn("Unable to delete OpenSearch index for dataset: " + datasetId);
}
}
return Response.ok().build();
Expand All @@ -392,8 +402,10 @@
try {
User user = userService.findUserByEmail(authUser.getEmail());
var datasetIds = datasetService.findAllDatasetIds();
StreamingOutput indexResponse = elasticSearchService.indexDatasetIds(datasetIds, user);
return Response.ok(indexResponse, MediaType.APPLICATION_JSON).build();
StreamingOutput elasticSearchResponse =
elasticSearchService.indexDatasetIds(datasetIds, user);
StreamingOutput openSearchResponse = openSearchService.indexDatasetIds(datasetIds, user);

Check warning on line 407 in src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this unused "openSearchResponse" local variable.

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozngGmrRvUMRnyA9&open=AZsDozngGmrRvUMRnyA9&pullRequest=2765

Check warning on line 407 in src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this useless assignment to local variable "openSearchResponse".

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozngGmrRvUMRnyA8&open=AZsDozngGmrRvUMRnyA8&pullRequest=2765
return Response.ok(elasticSearchResponse, MediaType.APPLICATION_JSON).build();
} catch (Exception e) {
return createExceptionResponse(e);
}
Expand All @@ -405,7 +417,9 @@
public Response indexDataset(@Auth AuthUser authUser, @PathParam("datasetId") Integer datasetId) {
try {
User user = userService.findUserByEmail(authUser.getEmail());
return elasticSearchService.indexDataset(datasetId, user);
Response elasticSearchResponse = elasticSearchService.indexDataset(datasetId, user);
Response openSearchResponse = openSearchService.indexDataset(datasetId, user);

Check warning on line 421 in src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this useless assignment to local variable "openSearchResponse".

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozngGmrRvUMRnyA-&open=AZsDozngGmrRvUMRnyA-&pullRequest=2765

Check warning on line 421 in src/main/java/org/broadinstitute/consent/http/resources/DatasetResource.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this unused "openSearchResponse" local variable.

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozngGmrRvUMRnyA_&open=AZsDozngGmrRvUMRnyA_&pullRequest=2765
return elasticSearchResponse;
} catch (Exception e) {
return createExceptionResponse(e);
}
Expand Down Expand Up @@ -446,7 +460,7 @@
@Timed
public Response searchDatasetIndexStream(@Auth DuosUser duosUser, String query) {
try {
InputStream inputStream = elasticSearchService.searchDatasetsStream(query);
InputStream inputStream = openSearchService.searchDatasetsStream(query);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use openSearch for the V2 endpoint.

StreamingOutput stream = createStreamingOutput(inputStream);
return Response.ok(stream).build();
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,22 +56,25 @@
private final DatasetServiceDAO datasetServiceDAO;
private final GCSService gcsService;
private final ElasticSearchService elasticSearchService;
private final OpenSearchService openSearchService;
private final StudyDAO studyDAO;
private final EmailService emailService;

public DatasetRegistrationService(

Check warning on line 63 in src/main/java/org/broadinstitute/consent/http/service/DatasetRegistrationService.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Constructor has 8 parameters, which is greater than 7 authorized.

See more on https://sonarcloud.io/project/issues?id=DataBiosphere_consent&issues=AZsDozrbGmrRvUMRnyBG&open=AZsDozrbGmrRvUMRnyBG&pullRequest=2765
DatasetDAO datasetDAO,
DacDAO dacDAO,
DatasetServiceDAO datasetServiceDAO,
GCSService gcsService,
ElasticSearchService elasticSearchService,
OpenSearchService openSearchService,
StudyDAO studyDAO,
EmailService emailService) {
this.datasetDAO = datasetDAO;
this.dacDAO = dacDAO;
this.datasetServiceDAO = datasetServiceDAO;
this.gcsService = gcsService;
this.elasticSearchService = elasticSearchService;
this.openSearchService = openSearchService;
this.studyDAO = studyDAO;
this.emailService = emailService;
}
Expand Down Expand Up @@ -211,6 +214,15 @@
} catch (Exception e) {
logException(e);
}
try (Response response = openSearchService.indexDatasets(createdDatasetIds, user)) {
if (response.getStatus() >= 400) {
logWarn(
String.format(
"Error indexing datasets from registration: %s", registration.getStudyName()));
}
} catch (Exception e) {
logException(e);
}
return datasets;
}

Expand Down Expand Up @@ -280,6 +292,7 @@

Dataset updatedDataset = datasetDAO.findDatasetById(datasetId);
elasticSearchService.synchronizeDatasetInESIndex(updatedDataset, user, false);
openSearchService.synchronizeDatasetInESIndex(updatedDataset, user, false);
return updatedDataset;
}

Expand Down
Loading
Loading