From 23e9df12ba5ec3f8a7edbb164e39bd1971a189e7 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 4 Mar 2026 17:29:30 +0100 Subject: [PATCH 1/3] fix(indexing): Improve memory management for resource indexing --- .../search/service/converter/SearchDocumentConverter.java | 7 +++++-- src/main/java/org/folio/search/utils/JsonConverter.java | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java b/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java index 30382c677..33d324e31 100644 --- a/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java +++ b/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java @@ -82,8 +82,11 @@ private SearchDocumentBody convert(ConversionContext context) { var baseFields = convertMapUsingResourceFields(getNewAsMap(resourceEvent), resourceDescriptionFields, context); var searchFields = searchFieldsProcessor.getSearchFields(context); var resultDocument = mergeSafely(baseFields, searchFields); - return SearchDocumentBody.of(searchDocumentBodyConverter.apply(resultDocument), - indexingDataFormat, resourceEvent, INDEX); + // Release the large _new Map before serialization — resultDocument is already a fully independent copy. + // ResourceEvent is still referenced by SearchDocumentBody for id/tenant/resource metadata only. + resourceEvent.setNew(null); + var documentBody = searchDocumentBodyConverter.apply(resultDocument); + return SearchDocumentBody.of(documentBody, indexingDataFormat, resourceEvent, INDEX); } private List getResourceLanguages(List languageSource, Map resourceData) { diff --git a/src/main/java/org/folio/search/utils/JsonConverter.java b/src/main/java/org/folio/search/utils/JsonConverter.java index 719049d1f..0ba324bd1 100644 --- a/src/main/java/org/folio/search/utils/JsonConverter.java +++ b/src/main/java/org/folio/search/utils/JsonConverter.java @@ -220,7 +220,12 @@ public BytesArray toJsonBytes(Object value) { if (value == null) { return null; } - return new BytesArray(toJson(value)); + try { + return new BytesArray(objectMapper.writeValueAsBytes(value)); + } catch (JacksonException e) { + throw new SerializationException(String.format( + SERIALIZATION_ERROR_MSG_TEMPLATE, e.getMessage()), e); + } } /** From 1fd5f93e9ec77fb936a77e8f6967dc18c41bfff0 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 4 Mar 2026 18:55:53 +0100 Subject: [PATCH 2/3] Fix test --- src/test/java/org/folio/search/utils/JsonConverterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/folio/search/utils/JsonConverterTest.java b/src/test/java/org/folio/search/utils/JsonConverterTest.java index 90b8d2b11..1922c324c 100644 --- a/src/test/java/org/folio/search/utils/JsonConverterTest.java +++ b/src/test/java/org/folio/search/utils/JsonConverterTest.java @@ -67,7 +67,7 @@ void toJsonBytes_positive() throws JacksonException { var actual = jsonConverter.toJsonBytes(TestClass.of(FIELD_VALUE)); assertThat(actual).isEqualTo(JSON_BYTES_BODY); - verify(objectMapper).writeValueAsString(TestClass.of(FIELD_VALUE)); + verify(objectMapper).writeValueAsBytes(TestClass.of(FIELD_VALUE)); } @Test From 4d14de1f0151d527f80b42bae70f9626cfa19c0a Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Mon, 9 Mar 2026 19:40:30 +0100 Subject: [PATCH 3/3] Fix instance sharing logic being used on reindexing Improve memory management --- .../search/model/entity/CallNumberEntity.java | 97 ------------------- .../entity/InstanceCallNumberEntity.java | 17 ---- .../InstanceChildrenResourceService.java | 12 ++- .../MultiTenantSearchDocumentConverter.java | 23 +++++ .../impl/CallNumberResourceExtractor.java | 67 ++++++------- .../reindex/ReindexOrchestrationService.java | 3 +- .../reindex/jdbc/CallNumberRepository.java | 7 +- .../jdbc/ClassificationRepository.java | 39 +++----- .../reindex/jdbc/ContributorRepository.java | 42 ++++---- .../service/reindex/jdbc/ItemRepository.java | 6 +- .../reindex/jdbc/MergeInstanceRepository.java | 6 +- .../reindex/jdbc/SubjectRepository.java | 45 ++++----- .../reindex/jdbc/UploadRangeRepository.java | 13 --- ...ndexingInstanceCallNumberConsortiumIT.java | 2 + .../IndexingInstanceCallNumberIT.java | 6 +- .../impl/CallNumberResourceExtractorTest.java | 18 ++-- .../ReindexOrchestrationServiceTest.java | 23 +++-- .../reindex/jdbc/CallNumberRepositoryIT.java | 10 +- 18 files changed, 148 insertions(+), 288 deletions(-) delete mode 100644 src/main/java/org/folio/search/model/entity/CallNumberEntity.java delete mode 100644 src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java diff --git a/src/main/java/org/folio/search/model/entity/CallNumberEntity.java b/src/main/java/org/folio/search/model/entity/CallNumberEntity.java deleted file mode 100644 index 4d103fd3f..000000000 --- a/src/main/java/org/folio/search/model/entity/CallNumberEntity.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.folio.search.model.entity; - -import static org.apache.commons.lang3.StringUtils.truncate; - -import java.util.Objects; -import lombok.Getter; -import org.folio.search.utils.ShaUtils; -import org.jspecify.annotations.NonNull; - -@Getter -public class CallNumberEntity implements Comparable { - - public static final int CALL_NUMBER_MAX_LENGTH = 50; - private static final int CALL_NUMBER_PREFIX_MAX_LENGTH = 20; - private static final int CALL_NUMBER_SUFFIX_MAX_LENGTH = 25; - private static final int CALL_NUMBER_TYPE_MAX_LENGTH = 40; - - private final String id; - private final String callNumber; - private final String callNumberPrefix; - private final String callNumberSuffix; - private final String callNumberTypeId; - - CallNumberEntity(String id, String callNumber, String callNumberPrefix, String callNumberSuffix, - String callNumberTypeId) { - this.id = id; - this.callNumber = callNumber; - this.callNumberPrefix = callNumberPrefix; - this.callNumberSuffix = callNumberSuffix; - this.callNumberTypeId = callNumberTypeId; - } - - public static CallNumberEntityBuilder builder() { - return new CallNumberEntityBuilder(); - } - - @Override - public int hashCode() { - return Objects.hashCode(id); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CallNumberEntity that)) { - return false; - } - return Objects.equals(id, that.id); - } - - @Override - public int compareTo(@NonNull CallNumberEntity o) { - return id.compareTo(o.id); - } - - public static class CallNumberEntityBuilder { - private String id; - private String callNumber; - private String callNumberPrefix; - private String callNumberSuffix; - private String callNumberTypeId; - - CallNumberEntityBuilder() { } - - public CallNumberEntityBuilder id(String id) { - this.id = id; - return this; - } - - public CallNumberEntityBuilder callNumber(String callNumber) { - this.callNumber = truncate(callNumber, CALL_NUMBER_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberPrefix(String callNumberPrefix) { - this.callNumberPrefix = truncate(callNumberPrefix, CALL_NUMBER_PREFIX_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberSuffix(String callNumberSuffix) { - this.callNumberSuffix = truncate(callNumberSuffix, CALL_NUMBER_SUFFIX_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberTypeId(String callNumberTypeId) { - this.callNumberTypeId = truncate(callNumberTypeId, CALL_NUMBER_TYPE_MAX_LENGTH); - return this; - } - - public CallNumberEntity build() { - if (id == null) { - this.id = ShaUtils.sha(callNumber, callNumberPrefix, callNumberSuffix, callNumberTypeId); - } - return new CallNumberEntity(this.id, this.callNumber, this.callNumberPrefix, this.callNumberSuffix, - this.callNumberTypeId); - } - } -} diff --git a/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java b/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java deleted file mode 100644 index a50f3484e..000000000 --- a/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.folio.search.model.entity; - -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; - -@Getter -@Builder -@EqualsAndHashCode -public class InstanceCallNumberEntity { - - private String callNumberId; - private String itemId; - private String instanceId; - private String locationId; - private String tenantId; -} diff --git a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java index bb48a4b21..e8b3cb0a6 100644 --- a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java +++ b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java @@ -93,6 +93,16 @@ public void persistChildrenOnReindex(String tenantId, ResourceType resourceType, .tenant(tenantId) ._new(instance)) .toList(); - persistChildren(tenantId, resourceType, events); + + var extractors = resourceExtractors.get(resourceType); + if (extractors == null) { + return; + } + + var shared = consortiumTenantProvider.isCentralTenant(tenantId); + + // Process child resources normally + extractors.forEach(resourceExtractor -> + resourceExtractor.persistChildren(tenantId, shared, events)); } } diff --git a/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java b/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java index dd66f81e2..62d140185 100644 --- a/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java +++ b/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java @@ -57,6 +57,29 @@ public Map> convert(Collection r .collect(groupingBy(SearchDocumentBody::getResource)); } + /** + * Converts {@link ResourceEvent} objects to a flat list of {@link SearchDocumentBody} objects. + * All events are expected to carry the same tenant ID that matches the current execution context — + * per-tenant grouping and context switching are therefore skipped entirely. + * + * @param resourceEvents list with {@link ResourceEvent} objects, all belonging to the current tenant + * @return flat {@link List} of {@link SearchDocumentBody} objects + */ + public List convertForReindex(Collection resourceEvents) { + log.debug("convertForReindex:: by [resourceEvents.size: {}]", collectionToLogMsg(resourceEvents, true)); + + if (CollectionUtils.isEmpty(resourceEvents)) { + return List.of(); + } + + return resourceEvents.stream() + .flatMap(this::populateResourceEvents) + .map(event -> event.getId() != null ? event : event.id(getResourceEventId(event))) + .map(searchDocumentConverter::convert) + .flatMap(Optional::stream) + .toList(); + } + private List convertForTenant(Entry> entry) { var convert = (Supplier>) () -> entry.getValue().stream() diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java index b37dfbe43..dfe493627 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java @@ -2,26 +2,24 @@ import static org.apache.commons.collections4.MapUtils.getMap; import static org.apache.commons.collections4.MapUtils.getString; -import static org.folio.search.model.entity.CallNumberEntity.CALL_NUMBER_MAX_LENGTH; +import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.SearchConverterUtils.getNewAsMap; import static org.folio.search.utils.SearchUtils.prepareForExpectedFormat; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.domain.dto.TenantConfiguredFeature; -import org.folio.search.model.entity.CallNumberEntity; -import org.folio.search.model.entity.InstanceCallNumberEntity; import org.folio.search.model.types.ResourceType; import org.folio.search.service.FeatureConfigService; import org.folio.search.service.converter.preprocessor.extractor.ChildResourceExtractor; import org.folio.search.service.reindex.jdbc.CallNumberRepository; -import org.folio.search.utils.JsonConverter; +import org.folio.search.utils.ShaUtils; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -36,14 +34,16 @@ public class CallNumberResourceExtractor extends ChildResourceExtractor { public static final String SUFFIX_FIELD = "suffix"; public static final String TYPE_ID_FIELD = "typeId"; - private final JsonConverter jsonConverter; + private static final int CALL_NUMBER_MAX_LENGTH = 50; + private static final int CALL_NUMBER_PREFIX_MAX_LENGTH = 20; + private static final int CALL_NUMBER_SUFFIX_MAX_LENGTH = 25; + private static final int CALL_NUMBER_TYPE_MAX_LENGTH = 40; + private final FeatureConfigService featureConfigService; public CallNumberResourceExtractor(CallNumberRepository repository, - JsonConverter jsonConverter, FeatureConfigService featureConfigService) { super(repository); - this.jsonConverter = jsonConverter; this.featureConfigService = featureConfigService; } @@ -57,14 +57,12 @@ protected List> constructRelations(boolean shared, ResourceE List> entities) { var resourceMap = getNewAsMap(event); return entities.stream() - .map(entity -> InstanceCallNumberEntity.builder() - .callNumberId(getString(entity, "id")) - .itemId(getString(resourceMap, "id")) - .instanceId(getString(resourceMap, "instanceId")) - .locationId(getString(resourceMap, "effectiveLocationId")) - .tenantId(event.getTenant()) - .build()) - .map(jsonConverter::convertToMap) + .map(entity -> Map.of( + "callNumberId", getString(entity, "id"), + "itemId", getString(resourceMap, "id"), + "instanceId", getString(resourceMap, "instanceId"), + "locationId", getString(resourceMap, "effectiveLocationId"), + "tenantId", event.getTenant())) .toList(); } @@ -73,9 +71,24 @@ protected Map constructEntity(Map entityProperti if (entityProperties == null || !featureConfigService.isEnabled(TenantConfiguredFeature.BROWSE_CALL_NUMBERS)) { return Collections.emptyMap(); } - return toCallNumberEntity(entityProperties) - .map(jsonConverter::convertToMap) - .orElse(Collections.emptyMap()); + var callNumberComponents = getCallNumberComponents(entityProperties); + var callNumber = prepareForExpectedFormat( + getString(callNumberComponents, CALL_NUMBER_FIELD), CALL_NUMBER_MAX_LENGTH); + if (StringUtils.isBlank(callNumber)) { + return Collections.emptyMap(); + } + var callNumberPrefix = truncate(getString(callNumberComponents, PREFIX_FIELD), CALL_NUMBER_PREFIX_MAX_LENGTH); + var callNumberSuffix = truncate(getString(callNumberComponents, SUFFIX_FIELD), CALL_NUMBER_SUFFIX_MAX_LENGTH); + var callNumberTypeId = truncate(getString(callNumberComponents, TYPE_ID_FIELD), CALL_NUMBER_TYPE_MAX_LENGTH); + var id = ShaUtils.sha(callNumber, callNumberPrefix, callNumberSuffix, callNumberTypeId); + + var entity = new HashMap(); + entity.put("id", id); + entity.put("callNumber", callNumber); + entity.put("callNumberPrefix", callNumberPrefix); + entity.put("callNumberSuffix", callNumberSuffix); + entity.put("callNumberTypeId", callNumberTypeId); + return entity; } @Override @@ -88,22 +101,6 @@ protected Set> getChildResources(Map event) return Set.of(event); } - private Optional toCallNumberEntity(Map entityProperties) { - var callNumberComponents = getCallNumberComponents(entityProperties); - var callNumber = prepareForExpectedFormat(getString(callNumberComponents, CALL_NUMBER_FIELD), - CALL_NUMBER_MAX_LENGTH); - if (StringUtils.isNotBlank(callNumber)) { - var callNumberEntity = CallNumberEntity.builder() - .callNumber(callNumber) - .callNumberPrefix(getString(callNumberComponents, PREFIX_FIELD)) - .callNumberSuffix(getString(callNumberComponents, SUFFIX_FIELD)) - .callNumberTypeId(getString(callNumberComponents, TYPE_ID_FIELD)) - .build(); - return Optional.of(callNumberEntity); - } - return Optional.empty(); - } - @SuppressWarnings("unchecked") private Map getCallNumberComponents(Map entityProperties) { return (Map) getMap(entityProperties, EFFECTIVE_CALL_NUMBER_COMPONENTS_FIELD, diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 1390c3b17..911bfb978 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -1,6 +1,5 @@ package org.folio.search.service.reindex; -import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.message.FormattedMessage; @@ -75,7 +74,7 @@ public boolean process(ReindexRecordsEvent event) { private FolioIndexOperationResponse fetchRecordsAndIndexForUploadRange(ReindexRangeIndexEvent event) { try { var resourceEvents = uploadRangeService.fetchRecordRange(event); - var documents = documentConverter.convert(resourceEvents).values().stream().flatMap(Collection::stream).toList(); + var documents = documentConverter.convertForReindex(resourceEvents); return elasticRepository.indexResources(documents); } catch (Exception ex) { throw handleReindexUploadFailure(event, ex.getMessage()); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index f09d08f92..a6bb6d8dc 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -258,10 +258,6 @@ protected RowMapper> rowToMapMapper() { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { var callNumberMap = getCallNumberMap(rs); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - callNumberMap.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } callNumberMap.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); return callNumberMap; }; @@ -278,8 +274,7 @@ private Map getCallNumberMap(ResultSet rs) throws SQLException { callNumberMap.put(CALL_NUMBER_PREFIX_FIELD, getCallNumberPrefix(rs)); callNumberMap.put(CALL_NUMBER_SUFFIX_FIELD, callNumberSuffix); callNumberMap.put(CALL_NUMBER_TYPE_ID_FIELD, getCallNumberTypeId(rs)); - var subResources = jsonConverter.toJson(parseInstanceSubResources(getInstances(rs))); - var maps = jsonConverter.fromJsonToListOfMaps(subResources).stream().filter(Objects::nonNull).toList(); + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { callNumberMap.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 0b6d31b4c..0989b268e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -179,19 +179,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map classification = new HashMap<>(); - classification.put("id", getId(rs)); - classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); - classification.put("typeId", getTypeId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return classification; - }; + return (rs, rowNum) -> buildClassificationMap(rs); } @Override @@ -238,21 +226,26 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map classification = new HashMap<>(); - classification.put("id", getId(rs)); - classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); - classification.put("typeId", getTypeId(rs)); + var classification = buildClassificationMap(rs); classification.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return classification; }; } + private Map buildClassificationMap(ResultSet rs) throws SQLException { + Map classification = new HashMap<>(); + classification.put("id", getId(rs)); + classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); + classification.put("typeId", getTypeId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return classification; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index 372ffa2a1..9e3ee79df 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -158,20 +158,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map contributor = new HashMap<>(); - contributor.put("id", getId(rs)); - contributor.put("name", getName(rs)); - contributor.put("contributorNameTypeId", getNameTypeId(rs)); - contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return contributor; - }; + return (rs, rowNum) -> buildContributorMap(rs); } @Override @@ -245,22 +232,27 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map contributor = new HashMap<>(); - contributor.put("id", getId(rs)); - contributor.put("name", getName(rs)); - contributor.put("contributorNameTypeId", getNameTypeId(rs)); + var contributor = buildContributorMap(rs); contributor.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return contributor; }; } + private Map buildContributorMap(ResultSet rs) throws SQLException { + Map contributor = new HashMap<>(); + contributor.put("id", getId(rs)); + contributor.put("name", getName(rs)); + contributor.put("contributorNameTypeId", getNameTypeId(rs)); + contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return contributor; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java index a89a6f47b..62f8690c4 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java @@ -3,7 +3,6 @@ import static org.folio.search.utils.JdbcUtils.getFullTableName; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.folio.search.configuration.properties.SearchConfigurationProperties; @@ -80,7 +79,7 @@ public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp, St private RowMapper> itemRowMapper() { return (rs, rowNum) -> { - Map item = new HashMap<>(); + var item = jsonConverter.fromJsonToMap(rs.getString("json")); item.put("id", rs.getString("id")); item.put("tenantId", rs.getString("tenant_id")); item.put("instanceId", rs.getString("instance_id")); @@ -88,9 +87,6 @@ private RowMapper> itemRowMapper() { item.put("isDeleted", rs.getBoolean("is_deleted")); item.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - var jsonContent = jsonConverter.fromJsonToMap(rs.getString("json")); - item.putAll(jsonContent); - return item; }; } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java index 928a09000..67e3d0c4e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java @@ -3,7 +3,6 @@ import static org.folio.search.utils.JdbcUtils.getFullTableName; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.log4j.Log4j2; @@ -110,7 +109,7 @@ public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp, St private RowMapper> instanceRowMapper() { return (rs, rowNum) -> { - Map instance = new HashMap<>(); + var instance = jsonConverter.fromJsonToMap(rs.getString("json")); instance.put("id", rs.getString("id")); instance.put("tenantId", rs.getString("tenant_id")); instance.put("shared", rs.getBoolean("shared")); @@ -118,9 +117,6 @@ private RowMapper> instanceRowMapper() { instance.put("isDeleted", rs.getBoolean("is_deleted")); instance.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - var jsonContent = jsonConverter.fromJsonToMap(rs.getString("json")); - instance.putAll(jsonContent); - return instance; }; } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index e199aa9b7..53dddbc39 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -186,21 +186,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map subject = new HashMap<>(); - subject.put("id", getId(rs)); - subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); - subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - subject.put("sourceId", getSourceId(rs)); - subject.put("typeId", getTypeId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return subject; - }; + return (rs, rowNum) -> buildSubjectMap(rs); } @Override @@ -249,23 +235,28 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map subject = new HashMap<>(); - subject.put("id", getId(rs)); - subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); - subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - subject.put("sourceId", getSourceId(rs)); - subject.put("typeId", getTypeId(rs)); + var subject = buildSubjectMap(rs); subject.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return subject; }; } + private Map buildSubjectMap(ResultSet rs) throws SQLException { + Map subject = new HashMap<>(); + subject.put("id", getId(rs)); + subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); + subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); + subject.put("sourceId", getSourceId(rs)); + subject.put("typeId", getTypeId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return subject; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java index 10e7e46df..294c71f6e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java @@ -13,13 +13,10 @@ import java.sql.Timestamp; import java.time.Instant; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import org.folio.search.configuration.properties.ReindexConfigurationProperties; -import org.folio.search.model.index.InstanceSubResource; import org.folio.search.model.reindex.UploadRangeEntity; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ReindexRangeStatus; @@ -28,7 +25,6 @@ import org.folio.spring.FolioExecutionContext; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; -import tools.jackson.core.type.TypeReference; public abstract class UploadRangeRepository extends ReindexJdbcRepository { @@ -47,7 +43,6 @@ ON CONFLICT (id) private static final String SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL = "SELECT * FROM %s WHERE entity_type = ?;"; private static final String DELETE_UPLOAD_RANGE_SQL = "DELETE FROM %s WHERE entity_type = ?;"; - private static final TypeReference> VALUE_TYPE_REF = new TypeReference<>() { }; protected final ReindexConfigurationProperties reindexConfig; @@ -90,14 +85,6 @@ protected String rangeTable() { protected abstract RowMapper> rowToMapMapper(); - protected Set parseInstanceSubResources(String instancesJson) { - try { - return jsonConverter.fromJson(instancesJson, VALUE_TYPE_REF); - } catch (Exception e) { - throw new IllegalArgumentException(e); - } - } - protected List createRanges() { var uploadRangeLevel = reindexConfig.getUploadRangeLevel(); return RangeGenerator.createHexRanges(uploadRangeLevel); diff --git a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java index ff6a41564..487ef50e0 100644 --- a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java +++ b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java @@ -36,6 +36,7 @@ class IndexingInstanceCallNumberConsortiumIT extends BaseIntegrationTest { private static final String INSTANCE_ID = randomId(); + private static final String LOCATION_ID = randomId(); private static final String INSTANCE_TITLE = "title"; private static final String CALL_NUMBER = "test"; @@ -94,6 +95,7 @@ private static void assertInstanceCallNumberTenantId(String expectedTenantId, bo private static void setUpTestData() { var holdings = new Holding().id(randomId()); var item = new Item().id(randomId()).holdingsRecordId(holdings.getId()) + .effectiveLocationId(LOCATION_ID) .effectiveCallNumberComponents(new ItemEffectiveCallNumberComponents().callNumber(CALL_NUMBER)); var instance = new Instance().id(INSTANCE_ID).title(INSTANCE_TITLE).source("FOLIO") .holdings(List.of(holdings)) diff --git a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java index d9ec02caf..ba77a0873 100644 --- a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java +++ b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java @@ -34,6 +34,7 @@ class IndexingInstanceCallNumberIT extends BaseIntegrationTest { private static final String INSTANCE_ID_1 = randomId(); private static final String INSTANCE_ID_2 = randomId(); + private static final String LOCATION_ID = randomId(); @BeforeAll static void prepare() { @@ -77,7 +78,7 @@ void shouldIndexInstanceCallNumber_createNewDocument_onItemCreate() { // assert that the document contains the expected fields assertCallNumberDocFields(sourceAsMap); - // assert that the document contains the expected instances object with count 2 + // assert that the document contains the expected instances object with count 1 @SuppressWarnings("unchecked") var instances = (List>) sourceAsMap.get("instances"); assertThat(instances) @@ -137,7 +138,8 @@ private void assertCallNumberDocFields(Map sourceAsMap) { } private Item getItem(String itemId) { - return new Item().id(itemId).holdingsRecordId(randomId()).effectiveCallNumberComponents(callNumberComponents()); + return new Item().id(itemId).holdingsRecordId(randomId()).effectiveLocationId(LOCATION_ID) + .effectiveCallNumberComponents(callNumberComponents()); } private ItemEffectiveCallNumberComponents callNumberComponents() { diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java index 3e7bc2634..28e46936f 100644 --- a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java @@ -17,17 +17,14 @@ import java.util.function.Supplier; import org.folio.search.domain.dto.TenantConfiguredFeature; import org.folio.search.service.FeatureConfigService; -import org.folio.search.service.consortium.ConsortiumTenantProvider; import org.folio.search.service.reindex.jdbc.CallNumberRepository; -import org.folio.search.utils.JsonConverter; import org.folio.spring.testing.type.UnitTest; import org.folio.support.base.ChildResourceExtractorTestBase; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import tools.jackson.databind.json.JsonMapper; @UnitTest @ExtendWith(MockitoExtension.class) @@ -37,9 +34,8 @@ class CallNumberResourceExtractorTest extends ChildResourceExtractorTestBase { private CallNumberRepository repository; @Mock private FeatureConfigService configService; - @Mock - private ConsortiumTenantProvider tenantProvider; + @InjectMocks private CallNumberResourceExtractor extractor; @Override @@ -47,11 +43,6 @@ protected int getExpectedEntitiesSize() { return 1; } - @BeforeEach - void setUp() { - extractor = new CallNumberResourceExtractor(repository, new JsonConverter(new JsonMapper()), configService); - } - @Test void persistChildren() { when(configService.isEnabled(TenantConfiguredFeature.BROWSE_CALL_NUMBERS)).thenReturn(true); @@ -72,7 +63,10 @@ private static Supplier> callNumberBodySupplier() { SUFFIX_FIELD, "suffix", PREFIX_FIELD, "prefix", TYPE_ID_FIELD, "type-id" - ) + ), + "id", "id", + "instanceId", "instance-id", + "effectiveLocationId", "location-id" )); } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java index cb77c138d..b032f73e5 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java @@ -10,7 +10,6 @@ import static org.mockito.Mockito.when; import java.util.List; -import java.util.Map; import java.util.UUID; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.domain.dto.ResourceEvent; @@ -59,8 +58,8 @@ void process_shouldProcessSuccessfully() { .status(FolioIndexOperationResponse.StatusEnum.SUCCESS); when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); // Act @@ -69,7 +68,7 @@ void process_shouldProcessSuccessfully() { // Assert assertTrue(result); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).addProcessedUploadRanges(event.getEntityType(), 1); } @@ -84,15 +83,15 @@ void process_shouldThrowReindexException_whenElasticSearchReportsError() { .errorMessage("Error occurred during indexing."); when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -120,13 +119,13 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringDocumentConver var exceptionMessage = "Failed to convert documents"; when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenThrow(new RuntimeException(exceptionMessage)); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -138,15 +137,15 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringIndexing() { var exceptionMessage = "Failed to index documents in Elasticsearch"; when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java index a00d16a71..0c45707c1 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java @@ -108,13 +108,11 @@ void saveAll() { .extracting("callNumber", "instances") .contains( tuple("number1", - List.of(mapOf("count", null, "instanceContributors", null, - "instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), "instanceTitle", null, - "locationId", null, "resourceId", null, "shared", false, "tenantId", TENANT_ID, "typeId", null))), + List.of(mapOf("instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), + "locationId", null, "shared", false, "tenantId", TENANT_ID))), tuple("number2", - List.of(mapOf("count", null, "instanceContributors", null, - "instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), "instanceTitle", null, - "locationId", null, "resourceId", null, "shared", false, "tenantId", TENANT_ID, "typeId", null)))); + List.of(mapOf("instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), + "locationId", null, "shared", false, "tenantId", TENANT_ID)))); } @Test