Skip to content

[MOSIP-42126], [MOSIP-42180] - Performance fixes for packet manager #228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
NidhiKumari0201 wants to merge 3 commits into from
Closed

[MOSIP-42126], [MOSIP-42180] - Performance fixes for packet manager #228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0e956c
[MOSIP-42126] - Performance Improvement - Packet Manager (#213)
trialblazerseee Jul 7, 2025
e047442
[MOSIP-42126]-update pom version and maven central migration (#217)
tarique-azeez Jul 8, 2025
d69375f
MOSIP-42180 - Packet Manager Redis Cache Storing Duplicate BIometrics…
trialblazerseee Jul 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/push-trigger.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
secrets:
OSSRH_USER: ${{ secrets.OSSRH_USER }}
OSSRH_SECRET: ${{ secrets.OSSRH_SECRET }}
OSSRH_URL: ${{ secrets.OSSRH_SNAPSHOT_URL }}
OSSRH_URL: ${{ secrets.OSSRH_CENTRAL_URL }}
OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }}
GPG_SECRET: ${{ secrets.GPG_SECRET }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
Expand Down
58 changes: 24 additions & 34 deletions commons-packet/commons-packet-manager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<name>commons-packet-manager</name>
<description>Mosip commons project </description>
<url>https://github.com/mosip/commons</url>
<version>1.2.0.3</version>
<version>1.2.0.4-SNAPSHOT</version>
<properties>
<!-- maven -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand All @@ -23,6 +23,7 @@
<!-- <maven.sonar.plugin.version>3.7.0.1746</maven.sonar.plugin.version> -->
<maven.javadoc.version>3.2.0</maven.javadoc.version>
<maven-shade-plugin.version>2.3</maven-shade-plugin.version>
<central.publishing.maven.plugin.version>0.7.0</central.publishing.maven.plugin.version>

<!-- spring -->
<spring.boot.version>2.0.2.RELEASE</spring.boot.version>
Expand All @@ -46,7 +47,7 @@
<kernel-dataaccess-hibernate.version>1.2.0.1</kernel-dataaccess-hibernate.version>
<kernel.logger.logback.version>1.2.0.1</kernel.logger.logback.version>
<spring.boot.co>2.0.2.RELEASE</spring.boot.co>
<khazana.version>1.2.0.1</khazana.version>
<khazana.version>1.2.0.2-SNAPSHOT</khazana.version>
<sonar.coverage.exclusions>**/constants/**,**/config/**,**/audit/**,**/util/**,**/dto/**,**/entity/**,**/model/**,**/exception/**,**/repository/**,**/security/**,**/*Config.java,**/*BootApplication.java,**/*VertxApplication.java,**/cbeffutil/**,**/*Utils.java,**/*Validator.java,**/*Helper.java,**/verticle/**,**/VidWriter.java/**,**/masterdata/utils/**,**/spi/**,**/core/http/**,"**/LocationServiceImpl.java","**/RegistrationCenterMachineServiceImpl.java","**/RegistrationCenterServiceImpl.java","**/pridgenerator/**","**/idgenerator/prid","**/proxy/**","**/cryptosignature/**"</sonar.coverage.exclusions>
<sonar.cpd.exclusions>**/dto/**,**/entity/**,**/config/**</sonar.cpd.exclusions>
</properties>
Expand Down Expand Up @@ -154,18 +155,28 @@
</dependencies>


<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
<repository>
<id>ossrh</id>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<build>
<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>https://central.sonatype.com/repository/maven-snapshots/</url>
</snapshotRepository>
<repository>
<id>ossrh</id>
<url>https://central.sonatype.com/api/v1/publisher</url>
</repository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>${central.publishing.maven.plugin.version}</version>
<extensions>true</extensions>
<configuration>
<publishingServerId>ossrh</publishingServerId>
<autoPublish>false</autoPublish>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
Expand All @@ -188,27 +199,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.7</version>
<extensions>true</extensions>
<executions>
<execution>
<id>default-deploy</id>
<phase>deploy</phase>
<goals>
<goal>deploy</goal>
</goals>
</execution>
</executions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>false</autoReleaseAfterClose>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
Expand Down
6 changes: 4 additions & 2 deletions ...ket/commons-packet-manager/src/main/java/io/mosip/commons/packet/facade/PacketReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,13 @@ public Document getDocument(String id, String documentName, String source, Strin
* @param process : the process
* @return BiometricRecord : the biometric record
*/

// The caching is removed from this method and moved to the implementation class as part of MOSIP-42180 JIRA. If we use the older implementation class logic (Customized Packet Reader Implementation) then caching logic will not work because of newly introduced getBiometric() method in IPacketReader interface i.e. we need to implement the caching logic in new getBiometric() method in implementation class.
@PreAuthorize("hasRole('BIOMETRIC_READ')")
@Cacheable(value = "packets", key = "'biometrics'.concat('-').#id.concat('-').concat(#person).concat('-').concat(#modalities).concat('-').concat(#source).concat('-').concat(#process)", condition = "#bypassCache == false")
public BiometricRecord getBiometric(String id, String person, List<String> modalities, String source, String process, boolean bypassCache) {
LOGGER.info(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"getBiometric for source : " + source + " process : " + process);
return getProvider(source, process).getBiometric(id, person, modalities, source, process);
return getProvider(source, process).getBiometric(id, person, modalities, source, process, bypassCache);
}

/**
Expand Down Expand Up @@ -147,6 +148,7 @@ public Map<String, String> getMetaInfo(String id, String source, String process,
* @return
*/
@PreAuthorize("hasRole('DATA_READ')")
@Cacheable(value = "info", key = "{#id}")
public List<ObjectDto> info(String id) {
LOGGER.info(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"info called");
Expand Down
141 changes: 91 additions & 50 deletions ...t/commons-packet-manager/src/main/java/io/mosip/commons/packet/impl/PacketReaderImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@
import static io.mosip.commons.packet.constants.PacketManagerConstants.TYPE;
import static io.mosip.commons.packet.constants.PacketManagerConstants.VALUE;

import java.io.IOException;
import java.io.InputStream;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand All @@ -29,32 +27,28 @@
import org.apache.commons.lang3.StringUtils;
import org.assertj.core.util.Lists;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.ObjectMapper;

import io.mosip.commons.packet.constants.PacketManagerConstants;
import io.mosip.commons.packet.dto.Document;
import io.mosip.commons.packet.dto.Packet;
import io.mosip.commons.packet.dto.PacketInfo;
import io.mosip.commons.packet.exception.ApiNotAccessibleException;
import io.mosip.commons.packet.exception.GetAllIdentityException;
import io.mosip.commons.packet.exception.GetAllMetaInfoException;
import io.mosip.commons.packet.exception.GetBiometricException;
import io.mosip.commons.packet.exception.GetDocumentException;
import io.mosip.commons.packet.exception.PacketDecryptionFailureException;
import io.mosip.commons.packet.exception.PacketKeeperException;
import io.mosip.commons.packet.exception.PacketValidationFailureException;
import io.mosip.commons.packet.keeper.PacketKeeper;
import io.mosip.commons.packet.spi.IPacketReader;
import io.mosip.commons.packet.util.IdSchemaUtils;
import io.mosip.commons.packet.util.PacketManagerHelper;
import io.mosip.commons.packet.util.PacketManagerLogger;
import io.mosip.commons.packet.util.PacketValidator;
import io.mosip.commons.packet.util.ZipUtils;
Expand Down Expand Up @@ -91,6 +85,8 @@ public class PacketReaderImpl implements IPacketReader {
@Autowired
private PacketValidator packetValidator;

@Autowired
private CacheManager cacheManager;
/**
* Perform packet validations and audit errors. List of validations - 1. schema
* & idobject reference validation 2. files validation 3. decrypted packet
Expand Down Expand Up @@ -231,55 +227,18 @@ public Document getDocument(String id, String documentName, String source, Strin
}

@Override
public BiometricRecord getBiometric(String id, String biometricFieldName, List<String> modalities, String source,
String process) {
public BiometricRecord getBiometric(String id, String biometricFieldName, List<String> modalities, String source, String process, boolean byPassCache) {
LOGGER.info(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"getBiometric :: for - " + biometricFieldName);
"getBiometric :: for - " + biometricFieldName + " with byPassCache - " + byPassCache);
BiometricRecord biometricRecord = null;
String packetName = null;
String fileName = null;
try {
String bioString = packetReader.getField(id, biometricFieldName, source, process, false);//(String) idobjectMap.get(biometricFieldName);
JSONObject biometricMap = null;
if (bioString != null)
biometricMap = new JSONObject(bioString);
if (bioString == null || biometricMap == null || biometricMap.isNull(VALUE)) {
// biometric file not present in idobject. Search in meta data.
Map<String, String> metadataMap = getMetaInfo(id, source, process);
String operationsData = metadataMap.get(META_INFO_OPERATIONS_DATA);
if (StringUtils.isNotEmpty(operationsData)) {
JSONArray jsonArray = new JSONArray(operationsData);
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject jsonObject = (JSONObject) jsonArray.get(i);
if (jsonObject.has(LABEL)
&& jsonObject.get(LABEL).toString().equalsIgnoreCase(biometricFieldName)) {
packetName = ID;
fileName = jsonObject.isNull(VALUE) ? null : jsonObject.get(VALUE).toString();
break;
}
}
}
} else {
String idSchemaVersion = packetReader.getField(id,
idSchemaUtils.getIdschemaVersionFromMappingJson(), source, process, false);
Double schemaVersion = idSchemaVersion != null ? Double.valueOf(idSchemaVersion) : null;
packetName = idSchemaUtils.getSource(biometricFieldName, schemaVersion);
fileName = biometricMap.get(VALUE).toString();
}

if (packetName == null || fileName == null)
return null;

Packet packet = packetKeeper.getPacket(getPacketInfo(id, packetName, source, process));
InputStream biometrics = ZipUtils.unzipAndGetFile(packet.getPacket(), fileName);
if (biometrics == null)
return null;
BIR bir = CbeffValidator.getBIRFromXML(IOUtils.toByteArray(biometrics));
try {
BIR bir = loadBiometricsFromObjectStore(id, biometricFieldName, source, process, byPassCache);
biometricRecord = new BiometricRecord();
if(bir.getOthers() != null) {
HashMap<String, String> others = new HashMap<>();
bir.getOthers().entrySet().forEach(e -> {
others.put(e.getKey(), e.getValue());
others.put(e.getKey(), e.getValue());
});
biometricRecord.setOthers(others);
}
Expand All @@ -300,6 +259,88 @@ public BiometricRecord getBiometric(String id, String biometricFieldName, List<S
return biometricRecord;
}

// Kept for backward compatibility. This method will not utilize the cache. Will be removed in future
@Override
public BiometricRecord getBiometric(String id, String biometricFieldName, List<String> modalities, String source, String process) {
return getBiometric(id, biometricFieldName, modalities, source, process, false);
}

private String generateKey(String id, String biometricFieldName, String source, String process) {
return String.format("%s-%s-%s-%s", id, biometricFieldName, source, process);
}

private BIR loadBiometricsFromObjectStore(String id, String biometricFieldName, String source, String process, boolean byPassCache) throws Exception {
String cacheKey = generateKey(id, biometricFieldName, source, process);
Cache cache = cacheManager.getCache("packets");

if(byPassCache || cache == null) {
LOGGER.debug(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"Skipping Cache due to byPassCache : " + byPassCache + " or IsCachePresent : " + (cache != null));
return loadBiometricsFromObjectStore(id, biometricFieldName, source, process);
}

BIR cachedValue = cache.get(cacheKey, BIR.class);
if(cachedValue != null) {
LOGGER.debug(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"Cache Found for the Key : " + cacheKey);
return cachedValue;
}

LOGGER.debug(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"Cache not found for the Key : " + cacheKey + " Loading biometrics from ObjectStore");
BIR bir = loadBiometricsFromObjectStore(id, biometricFieldName, source, process);
if(bir != null) {
LOGGER.debug(PacketManagerLogger.SESSIONID, PacketManagerLogger.REGISTRATIONID, id,
"Adding cache the Key : " + cacheKey);
cache.put(cacheKey, bir);
}

return bir;
}

private BIR loadBiometricsFromObjectStore(String id, String biometricFieldName, String source, String process) throws Exception {
String packetName = null;
String fileName = null;

String bioString = packetReader.getField(id, biometricFieldName, source, process, false);//(String) idobjectMap.get(biometricFieldName);
JSONObject biometricMap = null;
if (bioString != null)
biometricMap = new JSONObject(bioString);
if (bioString == null || biometricMap == null || biometricMap.isNull(VALUE)) {
// biometric file not present in idobject. Search in meta data.
Map<String, String> metadataMap = getMetaInfo(id, source, process);
String operationsData = metadataMap.get(META_INFO_OPERATIONS_DATA);
if (StringUtils.isNotEmpty(operationsData)) {
JSONArray jsonArray = new JSONArray(operationsData);
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject jsonObject = (JSONObject) jsonArray.get(i);
if (jsonObject.has(LABEL)
&& jsonObject.get(LABEL).toString().equalsIgnoreCase(biometricFieldName)) {
packetName = ID;
fileName = jsonObject.isNull(VALUE) ? null : jsonObject.get(VALUE).toString();
break;
}
}
}
} else {
String idSchemaVersion = packetReader.getField(id,
idSchemaUtils.getIdschemaVersionFromMappingJson(), source, process, false);
Double schemaVersion = idSchemaVersion != null ? Double.valueOf(idSchemaVersion) : null;
packetName = idSchemaUtils.getSource(biometricFieldName, schemaVersion);
fileName = biometricMap.get(VALUE).toString();
}

if (packetName == null || fileName == null)
return null;

Packet packet = packetKeeper.getPacket(getPacketInfo(id, packetName, source, process));
InputStream biometrics = ZipUtils.unzipAndGetFile(packet.getPacket(), fileName);
if (biometrics == null)
return null;

return CbeffValidator.getBIRFromXML(IOUtils.toByteArray(biometrics));
}

@Override
public Map<String, String> getMetaInfo(String id, String source, String process) {
Map<String, String> finalMap = new LinkedHashMap<>();
Expand Down
4 changes: 4 additions & 0 deletions ...acket/commons-packet-manager/src/main/java/io/mosip/commons/packet/spi/IPacketReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,8 @@ public interface IPacketReader {
public Map<String, String> getMetaInfo(String id, String source, String process);

public List<Map<String, String>> getAuditInfo(String id, String source, String process);

public default BiometricRecord getBiometric(String id, String biometricSchemaField, List<String> modalities, String source, String process, boolean byPassCache) {
return null;
};
}
11 changes: 9 additions & 2 deletions ...ns-packet-manager/src/test/java/io/mosip/commons/packet/test/facade/PacketReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.anyBoolean;

import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -24,6 +25,8 @@
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.test.util.ReflectionTestUtils;

import io.mosip.commons.khazana.dto.ObjectDto;
Expand All @@ -46,6 +49,9 @@
@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"})
public class PacketReaderTest {

@Mock
private CacheManager cacheManager;

@InjectMocks
private PacketReader packetReader = new PacketReader();

Expand Down Expand Up @@ -74,7 +80,8 @@ public void setup() {
allFields.put("phone", "1234567");

Mockito.when(packetReaderProvider.getAll(anyString(), anyString(), anyString())).thenReturn(allFields);

Cache mockCache = Mockito.mock(Cache.class);
Mockito.when(cacheManager.getCache("packets")).thenReturn(mockCache);
}

@Test
Expand Down Expand Up @@ -156,7 +163,7 @@ public void testGetBiometrics() {
BiometricRecord biometricRecord = new BiometricRecord();
biometricRecord.setSegments(birTypeList);

Mockito.when(packetReaderProvider.getBiometric(anyString(), anyString(), anyList(), anyString(), anyString())).thenReturn(biometricRecord);
Mockito.when(packetReaderProvider.getBiometric(anyString(), anyString(), anyList(), anyString(), anyString(), anyBoolean())).thenReturn(biometricRecord);

BiometricRecord result = packetReader.getBiometric(id, "individualBiometrics", Lists.newArrayList(), source, process, true);

Expand Down
Loading
Loading