Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .settings/org.eclipse.core.resources.prefs
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
eclipse.preferences.version=1
encoding//eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/staticDropFiles/testResults.php=UTF-8
encoding/<project>=UTF-8
25 changes: 21 additions & 4 deletions JenkinsJobs/AutomatedTests/integrationTests.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ pipeline {
extensions: [cloneOption(depth: 1, shallow: true, noTags: true), sparseCheckout([
[path: 'JenkinsJobs/buildConfigurations.json'],
[path: 'JenkinsJobs/shared/utilities.groovy'],
[path: 'scripts/releng'],
[path: 'eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles'],

])])
script {
def buildConfigurations = readJSON(file: 'JenkinsJobs/buildConfigurations.json')
Expand Down Expand Up @@ -104,20 +107,34 @@ pipeline {
localResultsDirectory="workarea/${buildId}/eclipse-testing/results"
ssh [email protected] mkdir -p ${remoteResultsDirectory}
if [ -d "${localResultsDirectory}/xml" ] && [ -z "$(find ${localResultsDirectory}/xml -maxdepth 0 -empty)" ]; then

# As long as not all tests run with Java-25 (or later) we cannot use multi-file Source Programs and have to compile manually
pushd scripts/releng
javac TestResultsGenerator.java
javac utilities/JSON.java
javac utilities/OS.java
javac utilities/XmlProcessorFactoryRelEng.java
# Generate new summary file
java \
-DxmlDirectory="${WORKSPACE}/${localResultsDirectory}/xml" \
-DtestsConfigExpected=${JOB_BASE_NAME} \
-DtestManifestFile="${WORKSPACE}/eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml" \
TestResultsGenerator
popd

# First delete result files from a previous run of the same configuration (in case that test configuration was run again), then transfer the new results.
ssh [email protected] rm -rfv ${remoteResultsDirectory}/${JOB_BASE_NAME}* ${remoteResultsDirectory}/*/*${JOB_BASE_NAME}*
scp -r ${localResultsDirectory}/* [email protected]:${remoteResultsDirectory}

# Download the result summary directly to the storage server
ssh [email protected] \
curl --fail --location --output "${remoteResultsDirectory}/${JOB_BASE_NAME}.xml" "${BUILD_URL}/testReport/api/xml?tree=failCount,passCount,skipCount,duration"
curl --fail --location --output "${remoteResultsDirectory}/${JOB_BASE_NAME}.xml" "${BUILD_URL}/testReport/api/json?tree=failCount,passCount,skipCount,duration"
fi
'''
}
}
if (currentBuild.result == 'SUCCESS' || currentBuild.result == 'UNSTABLE') {
build job: 'Releng/updateTestResultIndex', wait: false, parameters: [
string(name: 'buildID', value: "${params.buildId}")
]
build job: 'Releng/updateIndex', wait: false
}
}
}
Expand Down
93 changes: 47 additions & 46 deletions JenkinsJobs/Builds/build.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ pipeline {
environment {
MAVEN_OPTS = '-Xmx4G'
CJE_ROOT = "${WORKSPACE}/cje-production"
SCRIPTS = "${WORKSPACE}/scripts"
AGG_DIR = "${CJE_ROOT}/gitCache/eclipse.platform.releng.aggregator"
logDir = "$CJE_ROOT/buildlogs"

Expand Down Expand Up @@ -168,31 +169,38 @@ pipeline {
}
// Git log creation
sh '''#!/bin/bash -xe
reportTimestamp=$(TZ="America/New_York" date +%Y%m%d-%H%M)
reportTimestamp=$(TZ='UTC' date +%Y%m%d-%H%M)
export lastTag=$(cat "${WORKSPACE}/lastTag")
if [[ -n "$lastTag" ]]; then
export gitLogFile="${DROP_DIR}/$BUILD_ID/gitLog.html"
export gitLogFile="${DROP_DIR}/$BUILD_ID/gitLog.json"
mkdir -p $(dirname "${gitLogFile}")

echo -e "<h2>Git log from $lastTag (previous) to $BUILD_ID (current)</h2>" > $gitLogFile
echo -e "<h2>The tagging, and this report, were done at about $reportTimestamp</h2>" >> $gitLogFile
echo '{' > $gitLogFile
echo "\\"label\\": \\"${BUILD_ID}\\"," >> $gitLogFile
echo "\\"logFrom\\": \\"${lastTag}\\"," >> $gitLogFile
echo "\\"logTo\\": \\"${BUILD_ID}\\"," >> $gitLogFile
echo "\\"timestamp\\": \\"${reportTimestamp}\\"," >> $gitLogFile
echo "\\"repositories\\": [" >> $gitLogFile >> $gitLogFile

function createGitLog() {
gitURL=$(git config remote.origin.url | sed --expression 's,[email protected]:,https://github.com/,' | sed 's/\\.git$//')
gitName="${gitURL##*/}"
gitLog=$(git log $lastTag..$BUILD_ID --date=short --format=format:"<tr><td class=\"datecell\">%cd</td><td class=\"commitcell\"><a href=\"${gitURL}/commit/%H\">%s</a></td><td class=\"authorcell\">%aN</td></tr>")
#TODO: handle potential double-quotes in commit message?
gitLog=$(git log $lastTag..$BUILD_ID --date=short --format=format:"{ \\"date\\": \\"%cd\\", \\"message\\": \\"%s\\", \\"author\\": \\"%aN\\", \\"url\\": \\"${gitURL}/commit/%H\\"},")
if [ -n "$gitLog" ]; then
echo "Record changes in ${gitName}"
echo "<table><tbody> <tr><th class=\"cell\" colspan=\"3\">Repository: ${gitName}</th></tr>" >> $gitLogFile
echo "<tr> <th class=\"datecell\">Date</th> <th class=\"commitcell\">Commit message</th> <th class=\"authorcell\">Author</th> </tr>" >> $gitLogFile
echo "${gitLog}" >> $gitLogFile
echo "</tbody></table><br><br>" >> $gitLogFile
echo '' >> $gitLogFile
echo "{" >> $gitLogFile
echo "\\"name\\": \\"${gitName}\\"," >> $gitLogFile
echo "\\"commits\\": [" >> $gitLogFile
echo "${gitLog%,}" >> $gitLogFile
echo "]}," >> $gitLogFile
fi
}
export -f createGitLog
createGitLog
git submodule foreach 'createGitLog'

echo "{}]}" >> $gitLogFile # Append empty object to keep JSON valid
else
echo -e "\n\tGit log not generated because a reasonable previous tag could not be found." > $gitLogFile
fi
Expand Down Expand Up @@ -291,14 +299,18 @@ pipeline {
stage('Gather Eclipse parts') {
tools {
jdk 'temurin-jdk25-latest'
}
}
environment {
KEYRING = credentials('secret-subkeys-releng.asc')
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
}
steps {
dir("${DROP_DIR}/${BUILD_ID}") {
script {
utilities.copyStaticWebsiteFiles("${AGG_DIR}", 'eclipse/build')
}
}
sh '''#!/bin/bash -xe
mkdir -p ${DROP_DIR}/${BUILD_ID}
cp $CJE_ROOT/buildproperties.* ${DROP_DIR}/${BUILD_ID}
source $CJE_ROOT/buildproperties.shsource

Expand Down Expand Up @@ -441,20 +453,12 @@ pipeline {
bash ${CJE_ROOT}/scripts/produceChecksum.sh eclipse
popd

cp -r ${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/staticDropFiles/. ${DROP_DIR}/${BUILD_ID}

java \
-DisBuildTested=true \
-DbuildType=${BUILD_TYPE} \
-DdropTokenList='%repository%,%sdk%,%tests%,%runtime%,%jdtc%,%swt%' \
-DdropHtmlFileName=index.php \
-DxmlDirectoryName=${DROP_DIR}/${BUILD_ID}/testresults/xml \
-DdropDirectoryName=${DROP_DIR}/${BUILD_ID} \
-DdropTemplateFileName=${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/templateFiles/index.template.php \
-DcompileLogsDirectoryName=${DROP_DIR}/${BUILD_ID}/compilelogs/plugins \
-DtestManifestFileName=${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/testManifest.xml \
-DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED} \
${WORKSPACE}/scripts/releng/TestResultsGenerator.java
-DdropDirectory=${DROP_DIR}/${BUILD_ID} \
${SCRIPTS}/releng/BuildDropDataGenerator.java mainEclipse
java \
-DdropDirectory=${DROP_DIR}/${BUILD_ID} \
${SCRIPTS}/releng/CompilerSummaryGenerator.java
'''
}
}
Expand All @@ -479,14 +483,18 @@ pipeline {
stage('Gather Equinox parts') {
tools {
jdk 'temurin-jdk25-latest'
}
}
environment {
KEYRING = credentials('secret-subkeys-releng.asc')
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
}
steps {
dir("${EQUINOX_DROP_DIR}/${BUILD_ID}") {
script {
utilities.copyStaticWebsiteFiles("${AGG_DIR}", 'equinox/build')
}
}
sh '''#!/bin/bash -xe
mkdir -p ${EQUINOX_DROP_DIR}/${BUILD_ID}
cp $CJE_ROOT/buildproperties.* ${EQUINOX_DROP_DIR}/${BUILD_ID}
source $CJE_ROOT/buildproperties.shsource

Expand Down Expand Up @@ -518,20 +526,18 @@ pipeline {
cat $f
done

# Publish Equinox
# Build Equinox Launcher archives
pushd $CJE_ROOT
mkdir -p $ECLIPSE_BUILDER_DIR/equinox/$TMP_DIR
mkdir -p $CJE_ROOT/$TMP_DIR
$BASE_BUILDER_ECLIPSE_EXE \
-application org.eclipse.ant.core.antRunner \
-buildfile $ECLIPSE_BUILDER_DIR/equinox/helper.xml \
-buildfile ${ECLIPSE_BUILDER_DIR}/equinox/buildConfigs/equinox-launchers/build.xml \
-data $CJE_ROOT/$TMP_DIR/workspace-publishEquinox \
-DEBuilderDir=$ECLIPSE_BUILDER_DIR \
-DbuildDir=$BUILD_ID \
-DbuildId=$BUILD_ID \
-DbuildRepo=$PLATFORM_REPO_DIR \
-DequinoxPostingDirectory=${EQUINOX_DROP_DIR} \
-DeqpublishingContent=$ECLIPSE_BUILDER_DIR/equinox/publishingFiles \
-Dequinox.build.configs=$ECLIPSE_BUILDER_DIR/equinox/buildConfigs \
-Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \
-v \
Expand All @@ -542,19 +548,9 @@ pipeline {
bash ${CJE_ROOT}/scripts/produceChecksum.sh equinox
popd

cp -r ${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/staticDropFiles/. ${EQUINOX_DROP_DIR}/${BUILD_ID}

java \
-DisBuildTested=false \
-DbuildType=${BUILD_TYPE} \
-DdropTokenList='%equinox%,%framework%,%extrabundles%,%other%,%launchers%,%osgistarterkits%' \
-DdropHtmlFileName=index.php \
-DxmlDirectoryName=${DROP_DIR}/$BUILD_ID/testresults/xml \
-DdropDirectoryName=${EQUINOX_DROP_DIR}/${BUILD_ID} \
-DdropTemplateFileName=${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/templateFiles/index.template.php \
-DcompileLogsDirectoryName=${EQUINOX_DROP_DIR}/${BUILD_ID}/compilelogs/plugins \
-DtestManifestFileName=${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/testManifest.xml \
${WORKSPACE}/scripts/releng/TestResultsGenerator.java
-DdropDirectory=${EQUINOX_DROP_DIR}/${BUILD_ID} \
${SCRIPTS}/releng/BuildDropDataGenerator.java mainEquinox
'''
}
}
Expand Down Expand Up @@ -599,6 +595,9 @@ pipeline {
}
}
stage('Archive build logs') {
tools {
jdk 'temurin-jdk25-latest'
}
steps {
script {
// See https://github.com/jenkinsci/pipeline-stage-view-plugin/tree/master/rest-api#get-jobjob-namewfapiruns
Expand All @@ -607,7 +606,7 @@ pipeline {
for (jobStage in description['stages']) {
if (jobStage.status != 'IN_PROGRESS') {
def prefix = 's' + String.format('%03d', stageIndex++) // prefix with index to establish stable order
def logFileName = "${logDir}/" + prefix + '_' + jobStage.name.replace(' ', '_').replace('Declarative:_', '') + '.log'
def logFileName = "${DROP_DIR}/${BUILD_ID}/buildlogs/" + prefix + '_' + jobStage.name.replace(' ', '_').replace('Declarative:_', '') + '.log'
sh """
curl --fail --silent --output '${logFileName}' ${BUILD_URL}/pipeline-overview/log?nodeId=${jobStage.id}
# Test (efficiently) if the file starts with the no-logs message and if yes, ensure it doesn't contain more other lines
Expand All @@ -620,8 +619,10 @@ pipeline {
}
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
sh '''#!/bin/bash -xe
dropPath=${EP_ECLIPSE_DROPS}/${BUILD_ID}
scp -r $logDir/* [email protected]:${dropPath}/buildlogs/
java \
-DdropDirectory=${DROP_DIR}/${BUILD_ID} \
${SCRIPTS}/releng/BuildDropDataGenerator.java buildLogs
rsync -avzh ${DROP_DIR}/${BUILD_ID}/buildlogs/ [email protected]:${EP_ECLIPSE_DROPS}/${BUILD_ID}/buildlogs/
'''
}
}
Expand Down
4 changes: 1 addition & 3 deletions JenkinsJobs/Releng/promoteBuild.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,6 @@ pipeline {
-DnewPrevious-release.baseline="${RELEASE_P2_REPOSITORY}"
'''
utilities.replaceAllInFile('cje-production/buildproperties.txt', [
'API_PREV_REF_LABEL=".*?"' : "API_PREV_REF_LABEL=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
'PREVIOUS_RELEASE_VER=".*?"' : "PREVIOUS_RELEASE_VER=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
'PREVIOUS_RELEASE_REPO_ID=".*?"' : "PREVIOUS_RELEASE_REPO_ID=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
'BASEBUILD_ID=".*?"' : "BASEBUILD_ID=\"${DL_DROP_ID}\"",
Expand Down Expand Up @@ -371,11 +370,10 @@ EOF
BUILD_ID: "${newBuildLabel}",
BUILD_TYPE: "${DL_TYPE}",
BUILD_TYPE_NAME: "${newTypeName}",
BUILD_DIR_SEG: "${newDropID}",
EQ_BUILD_DIR_SEG: "${newDropID}",
]
def buildPropertiesPHP = sh(returnStdout: true, script: "ssh [email protected] cat ${sourcePath}/buildproperties.php")
for (entry in newProperties) {
//TODO: update this to modify the buildproperties.json with the new names/labels and updated filenames/size(?)
buildPropertiesPHP = buildPropertiesPHP.replaceFirst('\\$' + entry.key + ' = "[^"]+"', '\\$' + entry.key + ' = "' + entry.value + '"')
}
writeFile(file: 'buildproperties.php', text: buildPropertiesPHP)
Expand Down
Loading
Loading