Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public enum NormalizerStep {
/**
* Normalize PIDs. See {@link PidNormalizer}.
**/
NORMALIZE_PIDS(settings -> new PidNormalizer(new PidSchemeVocabularyCached()));
NORMALIZE_PIDS(settings -> new PidNormalizer(PidSchemeVocabularyCached.getInstance()));

private final ActionCreator actionCreator;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.net.URL;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -28,18 +29,17 @@ public final class PidSchemeVocabularyCached {
private static final String URI_SCHEME = "https://raw.githubusercontent.com/europeana/data-europeana-gateway/refs/heads/main/config/pid_directory.yaml";
private static final int MAX_IMPORT_RETRIES = 5;
private static final long RETRY_BACKOFF_MS = Duration.ofSeconds(1).toMillis();

private List<PidScheme> schemes = List.of();
private final ReentrantLock importCacheLock = new ReentrantLock();
private final String sourceUri;
private final ReentrantLock importCacheLock = new ReentrantLock();
private final AtomicReference<List<PidScheme>> schemes = new AtomicReference<>(List.of());
private volatile long lastSuccessfulImportTime = 0;

/**
* Instantiates new Pid scheme vocabulary cached.
*
* @throws NormalizationConfigurationException the normalization configuration exception
*/
public PidSchemeVocabularyCached() throws NormalizationConfigurationException {
private PidSchemeVocabularyCached() throws NormalizationConfigurationException {
this(URI_SCHEME);
}

Expand All @@ -49,7 +49,7 @@ public PidSchemeVocabularyCached() throws NormalizationConfigurationException {
* @param sourceUri the source uri
* @throws NormalizationConfigurationException the normalization configuration exception
*/
PidSchemeVocabularyCached(String sourceUri) throws NormalizationConfigurationException {
private PidSchemeVocabularyCached(String sourceUri) throws NormalizationConfigurationException {
if (sourceUri == null || sourceUri.isBlank()) {
throw new IllegalArgumentException("sourceUri must not be blank");
}
Expand All @@ -63,6 +63,15 @@ public PidSchemeVocabularyCached() throws NormalizationConfigurationException {
}
}

/**
* Gets instance.
*
* @return the instance
*/
public static PidSchemeVocabularyCached getInstance() {
return PidSchemeVocabularyCacheHelper.INSTANCE;
}

/**
* Attempt to match a PID against the vocabulary.
*
Expand Down Expand Up @@ -97,7 +106,7 @@ private List<PidScheme> getAllSchemesFromCache() throws NormalizationConfigurati
// 2. Slow path: the cache needs to refresh, use lock to prevent concurrent imports
refreshCacheIfNeeded();
}
return schemes;
return schemes.get();
}

/**
Expand All @@ -106,7 +115,7 @@ private List<PidScheme> getAllSchemesFromCache() throws NormalizationConfigurati
* @return the boolean
*/
private boolean isCacheValid() {
return !schemes.isEmpty() && (System.currentTimeMillis() - lastSuccessfulImportTime) <= IMPORT_CACHE_TTL_HOUR;
return !schemes.get().isEmpty() && (System.currentTimeMillis() - lastSuccessfulImportTime) <= IMPORT_CACHE_TTL_HOUR;
}

/**
Expand Down Expand Up @@ -141,10 +150,9 @@ private void importPidSchemesWithRetry() throws NormalizationConfigurationExcept
int attempt = 1;
boolean importSuccessful = false;
while (attempt <= MAX_IMPORT_RETRIES && !importSuccessful) {
attempt++;
try {
LOGGER.debug("Attempting to import PID schemes (attempt {}/{})", attempt, MAX_IMPORT_RETRIES);
schemes = List.copyOf(importPidSchemes());
schemes.set(List.copyOf(importPidSchemes()));
importSuccessful = true;
} catch (NormalizationConfigurationException exception) {
lastException = exception;
Expand All @@ -162,12 +170,13 @@ private void importPidSchemesWithRetry() throws NormalizationConfigurationExcept
LOGGER.error("PID scheme import failed after {} attempts", MAX_IMPORT_RETRIES, exception);
}
}
attempt++;
}

if (!importSuccessful) {
LOGGER.warn("All import attempts failed, falling back to stale cache (age: {} seconds)",
Duration.ofMillis(System.currentTimeMillis() - lastSuccessfulImportTime).toSeconds());
if (schemes.isEmpty()) {
if (schemes.get().isEmpty()) {
throw new NormalizationConfigurationException("Could not import PID schemes after " + MAX_IMPORT_RETRIES + " attempts",
lastException);
}
Expand Down Expand Up @@ -207,4 +216,21 @@ private List<PidScheme> importPidSchemes() throws NormalizationConfigurationExce
throw new NormalizationConfigurationException("Could not import PID schemes from remote source", e);
}
}

/**
* The type Pid scheme vocabulary cache helper.
*/
private static class PidSchemeVocabularyCacheHelper {

private static final PidSchemeVocabularyCached INSTANCE;

static {
try {
INSTANCE = new PidSchemeVocabularyCached();
} catch (NormalizationConfigurationException e) {
LOGGER.error("Failed to initialize PidSchemeVocabularyCached", e);
throw new IllegalStateException("Initialization of PidSchemeVocabularyCached failed.", e);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/
public class PidSchemeImportException extends Exception {

private static final long serialVersionUID = 1L;
private static final long serialVersionUID = -3254360258206690287L;

/**
* Constructor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import eu.europeana.normalization.util.NormalizationConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand All @@ -34,47 +35,26 @@

class PidSchemeVocabularyCachedTest {

private static PidSchemeVocabularyCached vocabulary;
private static final String TEST_PID_ARK = "ark:/12148/bpt6k279983";
private static final String TEST_PID_URN = "urn:nbn:nl:ui:29-8f66e0a8-b7c9-40a4-be28-54a7c0177061";
private static PidSchemeVocabularyCached vocabulary;
private static WireMockServer wireMockServer;

@BeforeAll
static void setUp() throws NormalizationConfigurationException, IOException {
String sourceUri = "http://metis-normalization-github.test/directory.yaml";
wireMockServer = new WireMockServer(wireMockConfig()
.dynamicPort()
.enableBrowserProxying(true)
.notifier(new ConsoleNotifier(true)));
wireMockServer.start();

JvmProxyConfigurer.configureFor(wireMockServer);

wireMockServer.stubFor(get(urlEqualTo("/directory.yaml"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("directory.yaml"))));
wireMockServer.stubFor(get(urlEqualTo("/scheme_a.rdf"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("scheme_a.rdf"))));
wireMockServer.stubFor(get(urlEqualTo("/scheme_b.rdf"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("scheme_b.rdf"))));

vocabulary = new PidSchemeVocabularyCached(sourceUri);
}

@AfterAll
static void tearDown() {
JvmProxyConfigurer.restorePrevious();
wireMockServer.stop();
private static PidSchemeVocabularyCached createTestVocabulary(String sourceUri) throws NormalizationConfigurationException {
try {
Constructor<PidSchemeVocabularyCached> constructor =
PidSchemeVocabularyCached.class.getDeclaredConstructor(String.class);
constructor.setAccessible(true);
return constructor.newInstance(sourceUri);
} catch (Exception e) {
throw new NormalizationConfigurationException("Failed to create test vocabulary instance", e);
}
}

private static String loadResourceContent(String value) throws IOException {
String resource = "";
try (InputStream inputStream = PidSchemeVocabularyCachedTest.class.getClassLoader().getResourceAsStream("pidTestSchemes/"+value)) {
try (InputStream inputStream = PidSchemeVocabularyCachedTest.class.getClassLoader()
.getResourceAsStream("pidTestSchemes/" + value)) {
resource = new String(Objects.requireNonNull(inputStream).readAllBytes());
}
return resource;
Expand Down Expand Up @@ -107,6 +87,41 @@ private static Stream<Arguments> matchPidSchemePaths() {
);
}

@BeforeAll
static void setUp() throws NormalizationConfigurationException, IOException {
String sourceUri = "http://metis-normalization-github.test/directory.yaml";

wireMockServer = new WireMockServer(wireMockConfig()
.dynamicPort()
.enableBrowserProxying(true)
.notifier(new ConsoleNotifier(true)));
wireMockServer.start();

JvmProxyConfigurer.configureFor(wireMockServer);

wireMockServer.stubFor(get(urlEqualTo("/directory.yaml"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("directory.yaml"))));
wireMockServer.stubFor(get(urlEqualTo("/scheme_a.rdf"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("scheme_a.rdf"))));
wireMockServer.stubFor(get(urlEqualTo("/scheme_b.rdf"))
.withHost(equalTo("metis-normalization-github.test"))
.atPriority(1)
.willReturn(ok().withBody(loadResourceContent("scheme_b.rdf"))));

// Create a new test-specific vocabulary instance with the test URL
vocabulary = createTestVocabulary(sourceUri);
}

@AfterAll
static void tearDown() {
JvmProxyConfigurer.restorePrevious();
wireMockServer.stop();
}

@ParameterizedTest
@MethodSource("providedPidSchemeInformation")
void testActualMatcherWithPidSchemes(String pidValue, String canonicalPid, String resolvablePid, String schemeId) {
Expand Down Expand Up @@ -311,7 +326,7 @@ void testDifferentPidFormats() {
}

@Test
void testSchemeConsistency() {
void testSchemeConsistency() {
// Given Multiple retrievals of matcher should produce consistent results
// When
PidMatchResult result1 = vocabulary.matchPid(TEST_PID_ARK);
Expand Down Expand Up @@ -341,7 +356,6 @@ void testPerformanceWithFrequentMatching() {
@Test
void testUrlPidFormatVariations() {


// When & Then Test different URL variations
// Test ARK with https URL
PidMatchResult result1 = vocabulary.matchPid("https://n2t.net/ark:/12148/bpt6k279983");
Expand Down