Migrate website from PHP to plain HTML with Java-Script and markdown

Create json files with all variables and define custom html tags that a
java-script replaces on the client

TODO:
- Complete the testsResults.html page
- The buildLogs.html page is not implemented, but mostly just listing
log files
- The testLogs.html page is not implemented, but mostly just listing log
files
- Generate TOC for non markdown files. Decide if we want to use the
plain/simple or 'high-constrast' style
- Fully discard markdown files
- Complete generation of data files in the build
- Implement Equinox page too
- Clean-up of unused elements
- Delete testresults folder to not keep them forever
- Delete old website files
- Replace images with high-res/svg icons (for later)

Fixes #2656

Author: HannesWell

.project

@@ -15,7 +15,7 @@
 			<type>30</type>
 			<matcher>
 				<id>org.eclipse.ui.ide.multiFilter</id>
-				<arguments>1.0-projectRelativePath-matches-true-true-(eclipse.platform.releng.prereqs.sdk|eclipse.platform.releng.tychoeclipsebuilder|oomph|products|sites)/.*</arguments>
+				<arguments>1.0-projectRelativePath-matches-true-true-(eclipse.platform.releng.prereqs.sdk|eclipse.platform.releng.tychoeclipsebuilder|oomph|products|scripts|sites)/.*</arguments>
 			</matcher>
 		</filter>
 	</filteredResources>

JenkinsJobs/AutomatedTests/integrationTests.jenkinsfile

@@ -109,7 +109,7 @@ pipeline {
 										scp -r ${localResultsDirectory}/* [email protected]:${remoteResultsDirectory}
 										# Download the result summary directly to the storage server
 										ssh [email protected] \
-											curl --fail --location --output "${remoteResultsDirectory}/${JOB_BASE_NAME}.xml" "${BUILD_URL}/testReport/api/xml?tree=failCount,passCount,skipCount,duration"
+											curl --fail --location --output "${remoteResultsDirectory}/${JOB_BASE_NAME}.xml" "${BUILD_URL}/testReport/api/json?tree=failCount,passCount,skipCount,duration"
 									fi
 								'''
 							}

JenkinsJobs/Builds/build.jenkinsfile

@@ -168,31 +168,38 @@ pipeline {
 					}
 					// Git log creation
 					sh '''#!/bin/bash -xe
-						reportTimestamp=$(TZ="America/New_York" date +%Y%m%d-%H%M)
+						reportTimestamp=$(TZ='UTC' date +%Y%m%d-%H%M)
 						export lastTag=$(cat "${WORKSPACE}/lastTag")
 						if [[ -n "$lastTag" ]]; then
-							export gitLogFile="${DROP_DIR}/$BUILD_ID/gitLog.html"
+							export gitLogFile="${DROP_DIR}/$BUILD_ID/gitLog.json"
 							mkdir -p $(dirname "${gitLogFile}")
 
-							echo -e "<h2>Git log from $lastTag (previous) to $BUILD_ID (current)</h2>" > $gitLogFile
-							echo -e "<h2>The tagging, and this report, were done at about $reportTimestamp</h2>" >> $gitLogFile
+							echo '{' > $gitLogFile
+							echo "\\"label\\": \\"${BUILD_ID}\\"," >> $gitLogFile
+							echo "\\"logFrom\\": \\"${lastTag}\\"," >> $gitLogFile
+							echo "\\"logTo\\": \\"${BUILD_ID}\\"," >> $gitLogFile
+							echo "\\"timestamp\\": \\"${reportTimestamp}\\"," >> $gitLogFile
+							echo "\\"repositories\\": [" >> $gitLogFile >> $gitLogFile
 
 							function createGitLog() {
 								gitURL=$(git config remote.origin.url | sed --expression 's,[email protected]:,https://github.com/,' | sed 's/\\.git$//')
 								gitName="${gitURL##*/}"
-								gitLog=$(git log $lastTag..$BUILD_ID --date=short --format=format:"<tr><td class=\"datecell\">%cd</td><td class=\"commitcell\"><a href=\"${gitURL}/commit/%H\">%s</a></td><td class=\"authorcell\">%aN</td></tr>")
+								#TODO: handle potential double-quotes in commit message?
+								gitLog=$(git log $lastTag..$BUILD_ID --date=short --format=format:"{ \\"date\\": \\"%cd\\", \\"message\\": \\"%s\\", \\"author\\": \\"%aN\\", \\"url\\": \\"${gitURL}/commit/%H\\"},")
 								if [ -n "$gitLog" ]; then
 									echo "Record changes in ${gitName}"
-									echo "<table><tbody> <tr><th class=\"cell\" colspan=\"3\">Repository: ${gitName}</th></tr>" >> $gitLogFile
-									echo "<tr> <th class=\"datecell\">Date</th> <th class=\"commitcell\">Commit message</th> <th class=\"authorcell\">Author</th> </tr>" >> $gitLogFile
-									echo "${gitLog}" >> $gitLogFile
-									echo "</tbody></table><br><br>" >> $gitLogFile
-									echo '' >> $gitLogFile
+									echo "{" >> $gitLogFile
+									echo "\\"name\\": \\"${gitName}\\"," >> $gitLogFile
+									echo "\\"commits\\": [" >> $gitLogFile
+									echo "${gitLog%,}" >> $gitLogFile
+									echo "]}," >> $gitLogFile
 								fi
 							}
 							export -f createGitLog
 							createGitLog
 							git submodule foreach 'createGitLog'
+							
+							echo "{}]}" >> $gitLogFile # Append empty object to keep JSON valid
 						else
 							echo -e "\n\tGit log not generated because a reasonable previous tag could not be found." > $gitLogFile
 						fi
@@ -297,6 +304,9 @@ pipeline {
 				stage('Eclipse') {
 					stages {
 						stage('Gather Eclipse parts') {
+							tools {
+								jdk 'temurin-jdk25-latest'
+							}
 							environment {
 								KEYRING = credentials('secret-subkeys-releng.asc')
 								KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
@@ -450,27 +460,46 @@ pipeline {
 										cat $f
 									done
 
-									# Publish Eclipse
-									pushd $CJE_ROOT
-									$BASE_BUILDER_ECLIPSE_EXE \
+									pushd ${DROP_DIR}/${BUILD_ID}
+									export client=eclipse
+									bash ${ECLIPSE_BUILDER_DIR}/eclipse/extras/produceChecksum.sh
+									popd
+									
+									#TODO: call this like in tests directly and make eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml obsolete
+									#triggering ant runner
+									pushd ${EBuilderDir}/eclipse/publishingFiles
+									$BASE_BUILDER_ECLIPSE_EXE -debug \
 										-application org.eclipse.ant.core.antRunner \
-										-buildfile $ECLIPSE_BUILDER_DIR/eclipse/helper.xml \
+										-buildfile "${CJE_ROOT}/scripts/publish.xml" \
 										-data $CJE_ROOT/$TMP_DIR/workspace-publish \
-										-DAGGR_DIR=${AGG_DIR} \
-										-DcjeDir=$CJE_ROOT \
-										-DEBuilderDir=$ECLIPSE_BUILDER_DIR \
 										-DbuildDirectory=${DROP_DIR}/${BUILD_ID} \
-										-DbuildLabel=$BUILD_ID \
-										-DbuildDir=$BUILD_ID \
-										-DbuildRepo=$PLATFORM_REPO_DIR \
-										-DbuildType=$BUILD_TYPE \
-										-DpublishingContent=$ECLIPSE_BUILDER_DIR/eclipse/publishingFiles \
-										-DindexFileName=index.php \
-										-Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \
-										-v \
-										publish
-									popd
+										-DbuildType=${BUILD_TYPE} \
+										"-DdropTemplateFileName=${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/templateFiles/index.template.php" \
+										"-DcompileLogsDirectoryName=${DROP_DIR}/${BUILD_ID}/compilelogs/plugins" \
+										"-DindexFileName=index.php" \
+										"-DmanifestFile=${WORKSPACE}/git-repo/eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml" \
+										"-DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED}" \
+										generateIndex
+
+									# Prepare build drop website
+									cp -r sites/eclipse/build-page/. ${DROP_DIR}/${BUILD_ID}
+									java \
+										-Ddirectory=${DROP_DIR}/${BUILD_ID} \
+										scripts/releng/GenerateDropData.java mainEclipse
 								'''
+								script {
+									//TODO: Necessary?
+									def buildProperties = readBuildProperties("${CJE_ROOT}/buildproperties.properties")
+									def utilities = load "JenkinsJobs/shared/utilities.groovy"
+									dir("$CJE_ROOT/$DROP_DIR/$BUILD_IID") {
+										//TODO: Remove this and call the Java script instead
+										// def fileList = sh(script: "ssh [email protected] 'cd ${EP_ECLIPSE_DROPS}/I20251202-0300 && stat --format \"%n %s\" *'", returnStdout: true)
+										def fileList = sh(script: 'stat --format "%n %s" *', returnStdout: true)
+										def buildSiteProperties = generateSiteData(buildProperties, fileList)
+										utilities.writeJSON("buildproperties.json", buildSiteProperties)
+									}
+								}
+
 							}
 						}
 						stage('Promote Eclipse') {
@@ -492,6 +521,9 @@ pipeline {
 				stage('Equinox') {
 					stages {
 						stage('Gather Equinox parts') {
+							tools {
+								jdk 'temurin-jdk25-latest'
+							}
 							environment {
 								KEYRING = credentials('secret-subkeys-releng.asc')
 								KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
@@ -705,3 +737,23 @@ def exportPropertiesAsEnvironmentVariables(String propertiesFile, Collection<Str
 		assignEnvVariable(envVar, value.startsWith('"') && value.endsWith('"') ? value.substring(1, value.length() - 1) : value)
 	}
 }
+
+@NonCPS
+def generateSiteData(Map<String, String> buildProperties, String fileList) {
+	def files = fileList.split('\\R').collectEntries{ e -> e.split(' ') } //TODO: check for more than one space?
+	def testConfigurations = buildProperties.TEST_CONFIGURATIONS_EXPECTED.split(',').collect{ c ->
+		def config = c.split('-')
+		return "${config[2]}-${config[3]}-${config[4].substring(0, config[4].indexOf('_'))}".toString()
+	}
+	return [ identifier: buildProperties.BUILD_ID, label: buildProperties.BUILD_ID,
+			kind: buildProperties.BUILD_TYPE_NAME,
+			release: buildProperties.STREAM, releaseShort: buildProperties.RELEASE_VER,
+			expectedTests: testConfigurations,
+			p2Repository: [ name: '', size: '' ],
+			'eclipse-sdk-products': [ name: '', size: '' ],
+			'eclipse-platform-products': [ name: '', size: '' ],
+			'eclipse-tests': [ name: '', size: '' ],
+			'jdt-compiler': [ name: '', size: '' ],
+			'swt-binaries': [ name: '', size: '' ],
+			]
+}

JenkinsJobs/Releng/promoteBuild.jenkinsfile

@@ -215,7 +215,6 @@ pipeline {
 								-DnewPrevious-release.baseline="${RELEASE_P2_REPOSITORY}"
 						'''
 						utilities.replaceAllInFile('cje-production/buildproperties.txt', [
-							'API_PREV_REF_LABEL=".*?"' : "API_PREV_REF_LABEL=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
 							'PREVIOUS_RELEASE_VER=".*?"' : "PREVIOUS_RELEASE_VER=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
 							'PREVIOUS_RELEASE_REPO_ID=".*?"' : "PREVIOUS_RELEASE_REPO_ID=\"${BUILD_MAJOR}.${BUILD_MINOR}\"",
 							'BASEBUILD_ID=".*?"' : "BASEBUILD_ID=\"${DL_DROP_ID}\"",
@@ -371,11 +370,10 @@ EOF
 		BUILD_ID: "${newBuildLabel}",
 		BUILD_TYPE: "${DL_TYPE}",
 		BUILD_TYPE_NAME: "${newTypeName}",
-		BUILD_DIR_SEG: "${newDropID}",
-		EQ_BUILD_DIR_SEG: "${newDropID}",
 	]
 	def buildPropertiesPHP = sh(returnStdout: true, script: "ssh [email protected] cat ${sourcePath}/buildproperties.php")
 	for (entry in newProperties) {
+		//TODO: update this to modify the buildproperties.json with the new names/labels and updated filenames/size(?)
 		buildPropertiesPHP = buildPropertiesPHP.replaceFirst('\\$' + entry.key + ' = "[^"]+"', '\\$' + entry.key + ' = "' + entry.value + '"')
 	}
 	writeFile(file: 'buildproperties.php', text: buildPropertiesPHP)

JenkinsJobs/Releng/updateIndex.jenkinsfile

@@ -11,21 +11,177 @@ pipeline {
 	agent {
 		label 'basic'
 	}
+	environment {
+		GIT_REPO = "${WORKSPACE}/git-repo"
+		BUILD_SEPARATOR = 'BUILD_SEPARATOR'
+		// Download server paths
+		EP_ROOT = '/home/data/httpd/download.eclipse.org'
+		EP_ECLIPSE_DROPS = "${EP_ROOT}/eclipse/downloads/drops4"
+		EP_EQUINOX_DROPS = "${EP_ROOT}/equinox/drops"
+		EP_ECLIPSE_UPDATES = "${EP_ROOT}/eclipse/updates"
+	}
 	stages {
-		stage('Update index'){
+		stage('Update index') {
 			steps {
+				dir("${GIT_REPO}") {
+					checkout scmGit(userRemoteConfigs: [[url: "${scm.userRemoteConfigs[0].url}"]], branches: [[name: "${scm.branches[0].name}"]],
+						extensions: [cloneOption(depth: 1, shallow: true, noTags: true), sparseCheckout([
+						[path: 'JenkinsJobs/shared/utilities.groovy'],
+						[path: 'sites/eclipse'],
+					])])
+				}
+				sh '''
+					# Collect files of the overview site
+					mkdir site
+					pushd site
+					cp -r ${GIT_REPO}/sites/eclipse/overview-page/. .
+					# Copy in common shared files located in the folder of the indivdual build page
+					cp ${GIT_REPO}/sites/eclipse/build-page/page.css .
+					cp ${GIT_REPO}/sites/eclipse/build-page/page.js .
+					sed --in-place --expression='s|<script src="../build-page/page.js">|<script src="page.js">|g' index.html
+					popd
+				'''
+				script {
+					def utilities = load "${GIT_REPO}/JenkinsJobs/shared/utilities.groovy"
+					def buildDropsData;
+					sshagent(['projects-storage.eclipse.org-bot-ssh']) {
+						buildDropsData = sh(script: """
+							ssh [email protected] '
+								cd ${EP_ECLIPSE_DROPS}
+								for drop in \$(ls); do
+									if [ -d \${drop} ] && [ ! -f \${drop}/buildHidden ]; then
+										echo "\${drop}"
+										grep --count "expectedTestConfigs\\[]=" \${drop}/testConfigs.php
+										grep --count "foundTestConfigs\\[]=" \${drop}/testConfigsFound.php || echo 0
+										if [  -f \${drop}/buildUnstable ]; then
+											echo 'unstable'
+										fi
+										echo "${BUILD_SEPARATOR}"
+									fi
+								done
+							'
+							""".stripIndent(), returnStdout: true)
+					}
+					// TODO: Replace this by Java script and capture the ssh output in file passed to the script?
+					// The script could then also collect all the files?
+					def overviewData = parseBuildDrops(buildDropsData)
+					utilities.writeJSON('site/data.json', overviewData)
+				}
 				sshagent(['projects-storage.eclipse.org-bot-ssh']) {
-					sh '''#!/bin/bash -xe
-						PHP_PAGE="createIndex4x.php"
-						HTML_PAGE="index.html"
-						TEMP_INDEX="tempIndex.html"
-						
-						wget --no-verbose -O ${TEMP_INDEX} https://download.eclipse.org/eclipse/downloads/${PHP_PAGE} 2>&1
-						
-						scp ${TEMP_INDEX} [email protected]:/home/data/httpd/download.eclipse.org/eclipse/downloads/${HTML_PAGE}
+					sh '''
+						#TODO: adjust paths to real target
+						rsync -avzh site/. [email protected]:${EP_ROOT}/eclipse/try-outs/eclipse/downloads/
 					'''
+					// FIXME: remove the following
+					script {
+						def fileList = sh(script: "ssh [email protected] 'cd ${EP_ECLIPSE_DROPS}/I20251201-1800 && stat --format \"%n %s\" *'", returnStdout: true)
+						echo "fileList: ${fileList}"
+						def buildProperties = readBuildProperties(sh(script: "ssh [email protected] 'cat ${EP_ECLIPSE_DROPS}/I20251201-1800/buildproperties.properties'", returnStdout: true))
+						echo "buildProperties: ${buildProperties}"
+						def buildSiteProperties = generateSiteData(buildProperties, fileList)
+						echo "buildSiteProperties: ${buildSiteProperties}"
+					}
 				}
 			}
 		}
 	}
+	post {
+		always {
+			archiveArtifacts artifacts: 'site/**/*'
+		}
+	}
+}
+
+@NonCPS
+def parseBuildDrops(String buildDrops) {
+	def releases = []
+	def stableBuilds = []
+	def iBuilds = []
+	def yBuilds = []
+	for (dropData in buildDrops.trim().split("BUILD_SEPARATOR")) {
+		if (dropData.isEmpty()) {
+			continue;
+		}
+		def dropDataElements = dropData.trim().split('\\s+')
+		def String dropID = dropDataElements[0]
+		def int testsExpected = Integer.parseInt(dropDataElements[1])
+		def int testsFinished = Integer.parseInt(dropDataElements[2])
+		def boolean isUnstable = dropDataElements.length > 3 && dropDataElements[3] == 'unstable'
+		def boolean isOldStyleDrop = true
+		def drop = [ path: "drops4/${dropID}".toString(), status: isUnstable ? 'unstable' : 'success' ]
+		if (dropID.startsWith('R-')) {
+			drop.label = dropID.substring(2, dropID.lastIndexOf('-'))
+			drop.date = toUTCdateTime(dropID.substring(dropID.lastIndexOf('-') + 1))
+			releases += drop
+		} else if (dropID.startsWith('S-')) {
+			drop.label = dropID.substring(2, dropID.lastIndexOf('-'))
+			drop.date = toUTCdateTime(dropID.substring(dropID.lastIndexOf('-') + 1))
+			stableBuilds += drop
+		} else if (dropID.startsWith('I20')) {
+			drop.label = dropID
+			drop.date = toUTCdateTime(dropID.substring(1).replace('-', ''))
+			iBuilds += drop
+		} else if (dropID.startsWith('Y20')) {
+			drop.label = dropID
+			drop.date = toUTCdateTime(dropID.substring(1).replace('-', ''))
+			yBuilds += drop
+		}
+		drop.testsCompletion = "${testsFinished} of ${testsExpected}".toString()
+		drop.testsPath = isOldStyleDrop ? 'testResults.php' : 'testresults'
+	}
+	def overviewData = [releases: releases, stableBuilds: stableBuilds, iBuilds: iBuilds, yBuilds: yBuilds]
+	overviewData.values().each{ list -> list.sort({d1, d2 -> -d1.label.compareTo(d2.label)})}
+	// Select latest element among all releases, stable builds respectivly I-builds
+	return overviewData
+}
+
+@NonCPS
+def toUTCdateTime(String dateTime) {
+	if (dateTime.length() != 12) {
+		throw new IllegalArgumentException('Not an expected date-time string: ' + dateTime)
+	}
+	def int year = Integer.parseInt(dateTime.substring(0, 4))
+	def int month = Integer.parseInt(dateTime.substring(4, 6))
+	def int day = Integer.parseInt(dateTime.substring(6, 8))
+	def int hour = Integer.parseInt(dateTime.substring(8, 10))
+	def int minute = Integer.parseInt(dateTime.substring(10, 12))
+	return java.time.LocalDateTime.of(year, month, day, hour, minute)
+		.atZone(java.time.ZoneId.of("America/New_York"))
+		.withZoneSameInstant(java.time.ZoneOffset.UTC)
+		.toLocalDateTime().toString() + 'Z';
+}
+
+// FIXME: remove the following
+
+def readBuildProperties(String buildPropertiesFile) {
+	return readProperties(text: buildPropertiesFile, charset: 'UTF-8').collectEntries{n, v ->
+			v = v.trim();
+			return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)]
+		}
+}
+
+@NonCPS
+def generateSiteData(Map<String, String> buildProperties, String fileList) {
+	def testPrefix = "ep${buildProperties.RELEASE_VER.replace('.', '')}I-unit-"
+	def testConfigs = buildProperties.TEST_CONFIGURATIONS_EXPECTED.split(',').collect{ c ->
+				def config = c.split('-')
+				return "${config[2]}-${config[3]}-${config[4].substring(0, config[4].indexOf('_'))}"
+			}
+	echo "testConfigs: ${testConfigs}"
+	def res = [ identifier: buildProperties.BUILD_ID, label: buildProperties.BUILD_ID,
+			kind: buildProperties.BUILD_TYPE_NAME,
+			release: buildProperties.STREAM, releaseShort: buildProperties.RELEASE_VER,
+			expectedTests: buildProperties.TEST_CONFIGURATIONS_EXPECTED.split(',').collect{ c ->
+				def config = c.split('-')
+				return "${config[2]}-${config[3]}-${config[4].substring(0, config[4].indexOf('_'))}"
+			},
+			p2Repository: [ name: '', size: '' ],
+			'eclipse-sdk-products': [ name: '', size: '' ],
+			'eclipse-platform-products': [ name: '', size: '' ],
+			'eclipse-tests': [ name: '', size: '' ],
+			'jdt-compiler': [ name: '', size: '' ],
+			'swt-binaries': [ name: '', size: '' ],
+			]
+	echo "res: ${res}"
+	return res
 }

JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile

@@ -31,6 +31,7 @@ pipeline {
 				}
 			}
 		}
+		//TODO: can this job and all called (ANT) scripts become obsoelte?
 		stage('Update test results'){
 			environment {
 				// Download Server locations (seldomly change)