Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[4.x.x] Switch CI from Temurin to Liberica #5468

Merged
10 changes: 5 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ jobs:
jdk: ['8']
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v2
uses: actions/setup-java@v4
with:
distribution: temurin
distribution: liberica
java-version: ${{ matrix.jdk }}
- name: Cache local Ivy repository
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: ~/.ivy/cache
key: ${{ runner.os }}-ivy-${{ hashFiles('**/ivy.xml') }}
Expand All @@ -39,7 +39,7 @@ jobs:
run: ./build.sh test -Dtest.haltonerror=true -Dtest.haltonfailure=true
- name: Archive build logs
if: always()
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: ${{ runner.os }}-jdk${{ matrix.jdk }}-build-logs
retention-days: 5
Expand Down
14 changes: 14 additions & 0 deletions build/scripts/junit.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
depends="test-prepare, test-local, test-extensions, test-jetty, test-concurrent, test-remote, test-recovery, test-wrapup, test-ant"
description="Run jUnit tests"/>

<!-- Hide the progress bar of RecoveryManager.java when running the tests -->
<property name="exist.recovery.progressbar.hide" value="true"/>

<!-- jetty.port and jetty.port.ssl with defaults. /ljo -->
<property name="jetty.port" value="8080" />
<property name="jetty.port.ssl" value="8443" />
Expand Down Expand Up @@ -157,6 +160,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-local_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- local database, so should not be needed. /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -261,6 +265,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-jetty_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- fixme! - jetty.port.jetty /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -299,6 +304,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-concurrent_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- local database, so should not be needed. /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -328,6 +334,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-recovery_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- local database, so should not be needed. /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -388,6 +395,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-remote_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- fixme! - jetty.port.standalone /ljo -->
<sysproperty key="jetty.port" value="${jetty.port.standalone}" />
Expand Down Expand Up @@ -424,6 +432,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/test-ant_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- local database, so should not be needed. /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -453,6 +462,7 @@
<jvmarg line="${gc.log.file.prefix}${junit.reports.dat}/extra-tests_gc.log" />

<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- fixme! - jetty.port.jetty /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -522,6 +532,7 @@
</copy>
<java classname="org.exist.xquery.xqts.XQTS_To_junit">
<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="exist.jmxagent" value="" />
<!-- fixme! - jetty.port.jetty /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -551,6 +562,7 @@
<echo message="--------------"/>
<junit haltonfailure="${junit.haltonfailure}" haltonerror="${junit.haltonerror}" printsummary="on" showoutput="${junit.output}" fork="yes" maxmemory="${junit.forked.VM.maxmemory.external}">
<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>
<!-- fixme! - jetty.port.jetty /ljo -->
<sysproperty key="jetty.port" value="${jetty.port}" />
Expand Down Expand Up @@ -591,6 +603,7 @@
</delete>
<java classname="org.exist.xquery.xqts.QT3TS_To_junit">
<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="exist.jmxagent" value="" />
<classpath refid="classpath.core"/>
<classpath refid="classpath.aspectj"/>
Expand All @@ -608,6 +621,7 @@
<echo message="--------------"/>
<junit haltonfailure="${junit.haltonfailure}" haltonerror="${junit.haltonerror}" printsummary="on" showoutput="${junit.output}" fork="yes" maxmemory="${junit.forked.VM.maxmemory.external}">
<sysproperty key="exist.home" value="${basedir}" />
<sysproperty key="exist.recovery.progressbar.hide" value="${exist.recovery.progressbar.hide}"/>
<sysproperty key="java.util.logging.manager" value="org.apache.logging.log4j.jul.LogManager"/>

<classpath refid="classpath.core"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ public void addPlugin(final String className) {
// NOTE: must set interrupted flag
Thread.currentThread().interrupt();
}
LOG.error(e);
LOG.warn(e);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
import com.evolvedbinary.j8fu.function.SupplierE;
import org.exist.util.sanity.SanityCheck;

import javax.annotation.Nullable;

/**
* Database recovery. This class is used once during startup to check
* if the database is in a consistent state. If not, the class attempts to recover
Expand All @@ -59,11 +61,13 @@ public class RecoveryManager {
private final DBBroker broker;
private final JournalRecoveryAccessor journalRecovery;
private final boolean restartOnError;
private final boolean hideProgressBar;

public RecoveryManager(final DBBroker broker, final JournalManager journalManager, final boolean restartOnError) {
this.broker = broker;
this.journalRecovery = journalManager.getRecoveryAccessor(this);
this.restartOnError = restartOnError;
this.hideProgressBar = Boolean.getBoolean("exist.recovery.progressbar.hide");
}

/**
Expand Down Expand Up @@ -121,10 +125,10 @@ public boolean recover() throws LogException {
Lsn lastLsn = Lsn.LSN_INVALID;
Loggable next;
try {
final ProgressBar progress = new ProgressBar("Scanning journal ", FileUtils.sizeQuietly(last));
final long lastSize = FileUtils.sizeQuietly(last);
@Nullable final ProgressBar scanProgressBar = hideProgressBar ? null : new ProgressBar("Scanning journal ", lastSize);
while ((next = reader.nextEntry()) != null) {
// LOG.debug(next.dump());
progress.set(next.getLsn().getOffset());
if (next.getLogType() == LogEntryTypes.TXN_START) {
// new transaction starts: add it to the transactions table
txnsStarted.put(next.getTransactionId(), next);
Expand All @@ -136,7 +140,15 @@ public boolean recover() throws LogException {
lastCheckpoint = (Checkpoint) next;
}
lastLsn = next.getLsn();

if (scanProgressBar != null) {
scanProgressBar.set(next.getLsn().getOffset());
}
}

if (scanProgressBar != null) {
scanProgressBar.set(lastSize); // 100%
}
} catch (final LogException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Caught exception while reading log", e);
Expand All @@ -147,7 +159,7 @@ public boolean recover() throws LogException {
// if the last checkpoint record is not the last record in the file
// we need a recovery.
if ((lastCheckpoint == null || !lastCheckpoint.getLsn().equals(lastLsn)) &&
txnsStarted.size() > 0) {
!txnsStarted.isEmpty()) {
LOG.info("Dirty transactions: " + txnsStarted.size());
// starting recovery: reposition the log reader to the last checkpoint
if (lastCheckpoint == null)
Expand Down Expand Up @@ -251,10 +263,11 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader
// ------- REDO ---------
if (LOG.isInfoEnabled())
{LOG.info("First pass: redoing " + txnCount + " transactions...");}
final ProgressBar progress = new ProgressBar("Redo ", FileUtils.sizeQuietly(last));
Loggable next = null;
int redoCnt = 0;
try {
final long lastSize = FileUtils.sizeQuietly(last);
@Nullable final ProgressBar redoProgressBar = hideProgressBar ? null : new ProgressBar("Redo ", lastSize);
while ((next = reader.nextEntry()) != null) {
SanityCheck.ASSERT(next.getLogType() != LogEntryTypes.CHECKPOINT,
"Found a checkpoint during recovery run! This should not ever happen.");
Expand All @@ -272,9 +285,19 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader
// LOG.debug("Redo: " + next.dump());
// redo the log entry
next.redo();
progress.set(next.getLsn().getOffset());
if (next.getLsn().equals(lastLsn))
{break;} // last readable entry reached. Stop here.

if (redoProgressBar != null) {
redoProgressBar.set(next.getLsn().getOffset());
}

if (next.getLsn().equals(lastLsn)) {
// last readable entry reached. Stop here.
break;
}
}

if (redoProgressBar != null) {
redoProgressBar.set(lastSize); // 100% done
}
} catch (final Exception e) {
LOG.error("Exception caught while redoing transactions. Aborting recovery to avoid possible damage. " +
Expand All @@ -291,16 +314,19 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader
{LOG.info("Second pass: undoing dirty transactions. Uncommitted transactions: " +
runningTxns.size());}
// see if there are uncommitted transactions pending
if (runningTxns.size() > 0) {
if (!runningTxns.isEmpty()) {
// do a reverse scan of the log, undoing all uncommitted transactions
try {
final long lastSize = FileUtils.sizeQuietly(last);
@Nullable final ProgressBar undoProgressBar = hideProgressBar ? null : new ProgressBar("Undo ", lastSize);
while((next = reader.previousEntry()) != null) {
if (next.getLogType() == LogEntryTypes.TXN_START) {
if (runningTxns.get(next.getTransactionId()) != null) {
runningTxns.remove(next.getTransactionId());
if (runningTxns.size() == 0)
if (runningTxns.isEmpty()) {
// all dirty transactions undone
{break;}
break;
}
}
} else if (next.getLogType() == LogEntryTypes.TXN_COMMIT) {
// ignore already committed transaction
Expand All @@ -314,6 +340,14 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader
// LOG.debug("Undo: " + next.dump());
next.undo();
}

if (undoProgressBar != null) {
undoProgressBar.set(lastSize - next.getLsn().getOffset());
}
}

if (undoProgressBar != null) {
undoProgressBar.set(lastSize); // 100% done
}
} catch (final Exception e) {
LOG.warn("Exception caught while undoing dirty transactions. Remaining transactions " +
Expand Down