diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle index be40f16cca54..e5d557843d10 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle @@ -15,16 +15,19 @@ * limitations under the License. */ +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension + if (project != project.rootProject) { throw new GradleException("Applicable to rootProject only: " + project.path) } configure(rootProject) { - def urlVersion = rootProject.ext.baseVersion.replace('.', '_') + LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) + def urlVersion = buildGlobals.baseVersion.replace('.', '_') Provider luceneJavadocUrl = buildOptions.addOption("lucene.javadoc.url", "External Javadoc URL for documentation generator.", provider { - if (project.version != project.baseVersion) { + if (buildGlobals.snapshotBuild) { // non-release build does not cross-link between modules. return null } else { diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle index c6c9ba447518..7ecac2a28196 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle @@ -26,7 +26,8 @@ import com.vladsch.flexmark.parser.ParserEmulationProfile; import com.vladsch.flexmark.util.ast.Document; import com.vladsch.flexmark.util.data.MutableDataSet; import com.vladsch.flexmark.util.sequence.Escaping; -import groovy.text.SimpleTemplateEngine; +import groovy.text.SimpleTemplateEngine +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension configure(project(':lucene:documentation')) { tasks.register("markdownToHtml", Copy, { @@ -59,6 +60,13 @@ configure(project(':lucene:documentation')) { // list all properties used by the template here to allow uptodate checks to be correct: inputs.property('version', project.version) + def buildGlobals = project.extensions.getByType(LuceneBuildGlobalsExtension) + + binding.put("project", [ + "version": project.version, + "majorVersion": buildGlobals.majorVersion + ]) + binding.put('defaultCodecPackage', providers.provider { // static Codec defaultCodec = LOADER . lookup ( "LuceneXXX" ) ; def regex = ~/\bdefaultCodec\s*=\s*LOADER\s*\.\s*lookup\s*\(\s*"([^"]+)"\s*\)\s*;/ @@ -156,9 +164,7 @@ class MarkdownTemplateTask extends DefaultTask { @TaskAction void transform() { def engine = new SimpleTemplateEngine(); - def resolvedBinding = binding.get() + [ - project : project - ] + def resolvedBinding = new HashMap(binding.get()) String markdown = templateFile.withReader('UTF-8') { engine.createTemplate(it).make(resolvedBinding).toString(); } diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle index 5f72485a42f0..964ed04f1ee1 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle @@ -1,4 +1,5 @@ import org.gradle.internal.jvm.Jvm +import org.apache.lucene.gradle.plugins.java.RenderJavadocTaskBase /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -269,7 +270,7 @@ class OfflineLink implements Serializable { } @CacheableTask -class RenderJavadocTask extends DefaultTask { +abstract class RenderJavadocTask extends RenderJavadocTaskBase { @InputFiles @PathSensitive(PathSensitivity.RELATIVE) @IgnoreEmptyDirectories @@ -329,11 +330,6 @@ class RenderJavadocTask extends DefaultTask { @Optional ListProperty extraOpts = project.objects.listProperty(String) - @Optional - @Input - final Property executable = project.objects.property(String).convention( - project.provider { Jvm.current().javadocExecutable.toString() }) - @InputDirectory @PathSensitive(PathSensitivity.RELATIVE) @IgnoreEmptyDirectories diff --git a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle index d99daa91f6ff..7fa69f63aece 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle @@ -19,6 +19,9 @@ import org.apache.lucene.gradle.plugins.spotless.GoogleJavaFormatPlugin import org.apache.lucene.gradle.plugins.java.JavaFolderLayoutPlugin import org.apache.lucene.gradle.plugins.java.JavacConfigurationPlugin import org.apache.lucene.gradle.plugins.java.JarManifestConfigurationPlugin +import org.apache.lucene.gradle.plugins.java.TestsAndRandomizationPlugin +import org.apache.lucene.gradle.plugins.java.TestsBeastingPlugin +import org.apache.lucene.gradle.plugins.java.AlternativeJdkSupportPlugin allprojects { project -> if (['src/java', 'src/test'].any {path -> @@ -26,11 +29,12 @@ allprojects { project -> }) { plugins.apply("java-library") + plugins.apply(AlternativeJdkSupportPlugin) plugins.apply(JavaFolderLayoutPlugin) plugins.apply(JavacConfigurationPlugin) plugins.apply(JarManifestConfigurationPlugin) - - plugins.apply("lucene.java.tests-and-randomization") + plugins.apply(TestsAndRandomizationPlugin) + plugins.apply(TestsBeastingPlugin) plugins.apply("lucene.java.per-project-test-summary") plugins.apply("lucene.validation.forbidden-apis") diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.alternative-jdk-support.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.alternative-jdk-support.gradle deleted file mode 100644 index 888b8ea4adec..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.java.alternative-jdk-support.gradle +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.gradle.internal.jvm.JavaInfo -import org.gradle.internal.jvm.Jvm -import org.gradle.internal.jvm.inspection.JvmInstallationMetadata -import org.gradle.internal.jvm.inspection.JvmMetadataDetector -import org.gradle.jvm.toolchain.internal.InstallationLocation - -// This adds support for compiling and testing against a different Java runtime. -// -// I failed to set it up leveraging Gradle's toolchains because -// a toolchain spec is not flexible enough to provide an exact location of the JVM to be used; -// if you have two identical JVM lang. versions in auto-discovered JVMs, an arbitrary one is used (?). -// This situation is not uncommon when debugging low-level stuff (hand-compiled JVM binaries). - -if (project != project.rootProject) { - throw new GradleException("Applicable to rootProject only: " + project.path) -} - -Provider runtimeJavaHomeOption = - buildOptions.addDirOption("runtime.java.home", "Home directory path to an alternative compilation/ runtime JDK.") - -// we used to have "RUNTIME_JAVA_HOME" uppercase env variable support so keep this option too. -Provider legacyUpperCase = project.layout.projectDirectory.dir( - providers.environmentVariable("RUNTIME_JAVA_HOME") - ) -runtimeJavaHomeOption = runtimeJavaHomeOption.orElse(legacyUpperCase) - -JavaInfo jvmGradle = Jvm.current(); -JavaInfo jvmCurrent = { - if (runtimeJavaHomeOption.isPresent()) { - return Jvm.forHome(runtimeJavaHomeOption.get().asFile) - } else { - return jvmGradle - } -}() - -JvmMetadataDetector jvmDetector = project.services.get(JvmMetadataDetector) - -if (jvmGradle != jvmCurrent) { - configure(rootProject) { - tasks.register("altJvmWarning", { - doFirst { - def jvmInfo = { JavaInfo javaInfo -> - JvmInstallationMetadata jvmMetadata = jvmDetector.getMetadata(InstallationLocation.userDefined(javaInfo.javaHome, "specific path")) - return "${jvmMetadata.languageVersion} (${jvmMetadata.displayName} ${jvmMetadata.runtimeVersion}, home at: ${jvmMetadata.javaHome})" - } - - logger.warn("""NOTE: Alternative java toolchain will be used for compilation and tests: - Project will use ${jvmInfo(jvmCurrent)} - Gradle runs with ${jvmInfo(jvmGradle)} -""") - } - }) - } - - allprojects { - plugins.withType(JavaPlugin).configureEach { - // Any tests - tasks.withType(Test).configureEach { - dependsOn ":altJvmWarning" - executable = jvmCurrent.javaExecutable - } - - // Any javac compilation tasks - tasks.withType(JavaCompile).configureEach { - dependsOn ":altJvmWarning" - options.fork = true - options.forkOptions.javaHome = jvmCurrent.javaHome - } - - // Javadoc compilation. - def javadocExecutable = jvmCurrent.javadocExecutable - tasks.matching { it.name == "renderJavadoc" || it.name == "renderSiteJavadoc" }.configureEach { - dependsOn ":altJvmWarning" - executable = javadocExecutable.toString() - } - } - } -} - -// Expose these properties on the root project. We could use an extension here, it'd be nicer. -ext { - runtimeJavaExecutable = jvmCurrent.javaExecutable - runtimeJavaHome = jvmCurrent.javaHome - runtimeJavaVersion = jvmDetector.getMetadata(InstallationLocation.userDefined(jvmCurrent.javaHome, "specific path")).getLanguageVersion() - usesAltJvm = (jvmGradle != jvmCurrent); -} diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.beasting.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.beasting.gradle deleted file mode 100644 index b86917e5ad58..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.java.beasting.gradle +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This adds 'beast' task which clones tests a given number of times (preferably -// constrained with a filtering pattern passed via '--tests'). - -if (project != project.rootProject) { - throw new GradleException("Applicable to rootProject only: " + project.path) -} - -// TODO: subtasks are not run in parallel (sigh, gradle removed this capability for intra-project tasks). -// TODO: maybe it would be better to take a deeper approach and just feed the task -// runner duplicated suite names (much like https://github.com/gradle/test-retry-gradle-plugin) -// TODO: this is a somewhat related issue: https://github.com/gradle/test-retry-gradle-plugin/issues/29 - -def beastingMode = gradle.startParameter.taskNames.any { name -> name == 'beast' || name.endsWith(':beast') } -def rootSeedUserProvided = rootProject.ext.rootSeedUserProvided - -if (beastingMode) { - if (rootSeedUserProvided) { - rootProject.tasks.register("warnAboutConstantSeed", { - doFirst { - logger.warn("Root randomization seed is externally provided, all duplicated runs will use the same starting seed.") - } - }) - } -} - -allprojects { - plugins.withType(JavaPlugin).configureEach { - Provider dupsOption = buildOptions.addIntOption("tests.dups", "Reiterate runs of entire test suites this many times ('beast' task).") - - if (beastingMode) { - def beastTask = tasks.register("beast", BeastTask, { - description = "Run a test suite (or a set of tests) many times over (duplicate 'test' task)." - group = "Verification" - }) - - if (!dupsOption.isPresent()) { - throw new GradleException("Specify -Ptests.dups=[count] for beast task.") - } - - // generate N test tasks and attach them to the beasting task for this project; - // the test filter will be applied by the beast task once it is received from - // command line. - def subtasks = (1..dupsOption.get()).collect { value -> - return tasks.register("test_${value}", Test, { - failFast = true - doFirst { - // If there is a global root seed, use it (all duplicated tasks will run - // from the same starting seed). Otherwise pick a sequential derivative. - if (!rootSeedUserProvided) { - systemProperty("tests.seed", - String.format("%08X", new Random(rootProject.ext.rootSeedLong + value).nextLong())) - } - } - }) - } - - beastTask.configure { - dependsOn subtasks - } - } - } -} - -/** - * We have to declare a dummy task here to be able to reuse the same syntax for 'test' task - * filter option. - */ -abstract class BeastTask extends DefaultTask { - @Option(option = "tests", description = "Sets test class or method name to be included, '*' is supported.") - public void setTestNamePatterns(List patterns) { - taskDependencies.getDependencies(this).each { subtask -> - subtask.filter.setCommandLineIncludePatterns(patterns) - } - } - - @TaskAction - void run() { - } -} diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle index 9e3fcbd3fc06..4de4f5520714 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle @@ -16,7 +16,7 @@ */ import java.util.regex.Pattern -import org.apache.lucene.gradle.ErrorReportingTestListener +import org.apache.lucene.gradle.plugins.java.ErrorReportingTestListener // Display all failed tests at the end of the build. diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.tests-and-randomization.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.tests-and-randomization.gradle deleted file mode 100644 index 1b0ef262dbe1..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.java.tests-and-randomization.gradle +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.tools.ant.taskdefs.condition.Os -import org.apache.tools.ant.types.Commandline -import org.gradle.api.tasks.testing.logging.* -import com.carrotsearch.randomizedtesting.generators.RandomPicks -import org.apache.lucene.gradle.ErrorReportingTestListener -import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsExtension - -def loggingConfigFile = rootProject.layout.projectDirectory.file("gradle/testing/logging.properties") -def verboseModeHookInstalled = false - -// Pass certain build options to the test JVM as system properties -def optionsInheritedAsProperties = [] - -// JVM options -Provider minHeapSizeOption = buildOptions.addOption("tests.minheapsize", "Minimum heap size for test JVMs", "256m") -Provider heapSizeOption = buildOptions.addOption("tests.heapsize", "Heap size for test JVMs", "512m") - -// Vectorization related options. -String randomVectorSize = RandomPicks.randomFrom(new Random(project.ext.projectSeedLong), [ - "default", - "128", - "256", - "512" -]) -Provider defaultvectorizationOption = buildOptions.addBooleanOption("tests.defaultvectorization", - "Uses defaults for running tests with correct JVM settings to test Panama vectorization (tests.jvmargs, tests.vectorsize, tests.forceintegervectors).", false) -buildOptions.addOption("tests.vectorsize", "Sets preferred vector size in bits.", provider { - defaultvectorizationOption.get() ? 'default' : randomVectorSize -}) -buildOptions.addBooleanOption("tests.forceintegervectors", "Forces use of integer vectors even when slow.", provider { - defaultvectorizationOption.get() ? false : (randomVectorSize != 'default') -}) -optionsInheritedAsProperties += [ - "tests.vectorsize", - "tests.forceintegervectors" -] - -// Verbatim JVM arguments; make it also accept TEST_JVM_ARGS env. variable. -Provider jvmArgsOption = buildOptions.addOption("tests.jvmargs", "Arguments passed to each forked test JVM.", - provider { - -> - return (isCIBuild || defaultvectorizationOption.get()) ? "" : "-XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1" - }) -jvmArgsOption = project.providers.environmentVariable("TEST_JVM_ARGS").orElse(jvmArgsOption) - -// cwd and tmp dir for forked JVMs. -Provider workDirOption = buildOptions.addDirOption("tests.workDir", "Working directory for forked test JVMs.", - project.layout.buildDirectory.dir("tests-cwd")) -Provider tmpDirOption = buildOptions.addDirOption("tests.tmpDir", "Temp directory for forked test JVMs.", - project.layout.buildDirectory.dir("tests-tmp")) -def testsCwd = workDirOption.get().asFile -def testsTmpDir = workDirOption.get().asFile - -// Asserts, debug output. -Provider verboseOption = buildOptions.addBooleanOption("tests.verbose", "Enables verbose test output mode (emits full test outputs immediately).", false) -Provider haltOnFailureOption = buildOptions.addBooleanOption("tests.haltonfailure", "Halt processing early on test failure.", false) -Provider failFastOption = buildOptions.addBooleanOption("tests.failfast", "Stop the build early on failure.", false) -Provider rerunOption = buildOptions.addBooleanOption("tests.rerun", "Always rerun the test task, even if nothing has changed on input.", true) - -// How many testing JVM forks to create -Provider jvmsOption = buildOptions.addIntOption("tests.jvms", "The number of forked test JVMs", - provider { -> ((int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0))) }) - -// GITHUB#13986: Allow easier configuration of the Panama Vectorization provider with newer Java versions -Provider upperJavaFeatureVersionOption = buildOptions.addIntOption( - "tests.upperJavaFeatureVersion", "Min JDK feature version to configure the Panama Vectorization provider") - -// Test reiteration, filtering and component randomization options. - -// Propagate root seed so that it is visible and reported as an option in subprojects. -if (project != project.rootProject) { - buildOptions.addOption("tests.seed", "The \"root\" randomization seed for options and test parameters.", - rootProject.getExtensions().getByType(BuildOptionsExtension).getOption("tests.seed").asStringProvider()) -} -optionsInheritedAsProperties += ["tests.seed"] - -buildOptions.addIntOption("tests.iters", "Duplicate (re-run) each test case N times.") -optionsInheritedAsProperties += ["tests.iters"] - -buildOptions.addIntOption("tests.multiplier", "Value multiplier for randomized tests.") -optionsInheritedAsProperties += ["tests.multiplier"] - -buildOptions.addIntOption("tests.maxfailures", "Skip tests after a given number of failures.") -optionsInheritedAsProperties += ["tests.maxfailures"] - -buildOptions.addIntOption("tests.timeoutSuite", "Timeout (in millis) for an entire suite.") -optionsInheritedAsProperties += ["tests.timeoutSuite"] - -Provider assertsOption = buildOptions.addBooleanOption("tests.asserts", "Enables or disables assertions mode.", true) -optionsInheritedAsProperties += ["tests.asserts"] - -buildOptions.addBooleanOption("tests.infostream", "Enables or disables infostream logs.", false) -optionsInheritedAsProperties += ["tests.infostream"] - -buildOptions.addBooleanOption("tests.leaveTemporary", "Leave temporary directories after tests complete.", false) -optionsInheritedAsProperties += ["tests.leaveTemporary"] - -buildOptions.addOption("tests.codec", "Sets the codec tests should run with.", "random") -optionsInheritedAsProperties += ["tests.codec"] - -buildOptions.addOption("tests.directory", "Sets the Directory implementation tests should run with.", "random") -optionsInheritedAsProperties += ["tests.directory"] - -buildOptions.addOption("tests.postingsformat", "Sets the postings format tests should run with.", "random") -optionsInheritedAsProperties += ["tests.postingsformat"] - -buildOptions.addOption("tests.docvaluesformat", "Sets the doc values format tests should run with.", "random") -optionsInheritedAsProperties += ["tests.docvaluesformat"] - -buildOptions.addOption("tests.locale", "Sets the default locale tests should run with.", "random") -optionsInheritedAsProperties += ["tests.locale"] - -buildOptions.addOption("tests.timezone", "Sets the default time zone tests should run with.", "random") -optionsInheritedAsProperties += ["tests.timezone"] - -buildOptions.addOption("tests.filter", "Applies a test filter (see ./gradlew :helpTests).") -optionsInheritedAsProperties += ["tests.filter"] - -buildOptions.addBooleanOption("tests.nightly", "Enables or disables @Nightly tests.", false) -buildOptions.addBooleanOption("tests.monster", "Enables or disables @Monster tests.", false) -buildOptions.addBooleanOption("tests.awaitsfix", "Enables or disables @AwaitsFix tests.", false) -optionsInheritedAsProperties += [ - "tests.nightly", - "tests.monster", - "tests.awaitsfix" -] - -buildOptions.addBooleanOption("tests.gui", "Enables or disables @RequiresGUI tests.", provider { - -> - return rootProject.ext.isCIBuild -}) - -buildOptions.addOption("tests.file.encoding", "Sets the default file.encoding on test JVM.", provider { - -> - return RandomPicks.randomFrom(new Random(project.ext.projectSeedLong), [ - "US-ASCII", - "ISO-8859-1", - "UTF-8" - ]) -}) - -buildOptions.addBooleanOption("tests.faiss.run", "Explicitly run tests for the Faiss codec.", false) -optionsInheritedAsProperties += ["tests.faiss.run"] - -// TODO: do we still use these? -// Test data file used. -// [propName: 'tests.linedocsfile', value: 'europarl.lines.txt.gz', description: "Test data file path."], -// miscellaneous; some of them very weird. -// [propName: 'tests.LUCENE_VERSION', value: baseVersion, description: "Base Lucene version."], -// [propName: 'tests.bwcdir', value: null, description: "Data for backward-compatibility indexes."] - - -// If we're running in verbose mode and: -// 1) worker count > 1 -// 2) number of 'test' tasks in the build is > 1 -// then the output would very likely be mangled on the -// console. Fail and let the user know what to do. -def verboseMode = verboseOption.get().booleanValue() -if (verboseMode && !verboseModeHookInstalled) { - verboseModeHookInstalled = true - if (gradle.startParameter.maxWorkerCount > 1) { - gradle.taskGraph.whenReady { graph -> - def testTasks = graph.allTasks.findAll { task -> task instanceof Test } - if (testTasks.size() > 1) { - throw new GradleException("Run your tests in verbose mode only with --max-workers=1 option passed to gradle.") - } - } - } -} - -Provider minMajorVersion = upperJavaFeatureVersionOption.map { ver -> Integer.parseInt(JavaVersion.toVersion(ver).majorVersion) } -JavaVersion runtimeJava = rootProject.ext.runtimeJavaVersion -boolean incubatorJavaVersion = rootProject.ext.vectorIncubatorJavaVersions.contains(runtimeJava) - -// if the vector module is in incubator, pass lint flags to suppress excessive warnings. -if (incubatorJavaVersion) { - tasks.withType(JavaCompile).configureEach { - it.options.compilerArgs += ["-Xlint:-incubating"] - } -} - -tasks.withType(Test).configureEach { - // Running any test task should first display the root randomization seed. - dependsOn ":showTestsSeed" - - ext { - testOutputsDir = file("${reports.junitXml.outputLocation.get()}/outputs") - } - - // LUCENE-9660: Make it possible to always rerun tests, even if they're incrementally up-to-date. - if (rerunOption.get()) { - outputs.upToDateWhen { false } - } - - maxParallelForks = jvmsOption.get() - if (verboseMode && maxParallelForks != 1) { - logger.lifecycle("tests.jvm forced to 1 in verbose mode.") - maxParallelForks = 1 - } - - if (failFastOption.get()) { - failFast true - } - - workingDir testsCwd - useJUnit() - - minHeapSize = minHeapSizeOption.get() - maxHeapSize = heapSizeOption.get() - - ignoreFailures = (haltOnFailureOption.get() == false) - - // Up to JDK-15 we have to enforce --illegal-access=deny, because we want no code to access - // JDK internals; JDK-16 and later will default to deny, see https://openjdk.java.net/jeps/396: - if (rootProject.ext.runtimeJavaVersion < JavaVersion.VERSION_16) { - jvmArgs '--illegal-access=deny' - } - - if (assertsOption.get()) { - jvmArgs("-ea", "-esa") - } else { - enableAssertions = false - } - - // Lucene needs to optional modules at runtime, which we want to enforce for testing - // (if the runner JVM does not support them, it will fail tests): - jvmArgs '--add-modules', 'jdk.management' - - // dump heap on OOM. - jvmArgs "-XX:+HeapDumpOnOutOfMemoryError" - - // Enable the vector incubator module on supported Java versions: - - boolean manualMinMajorVersion = minMajorVersion.isPresent() && - Integer.parseInt(runtimeJava.majorVersion) <= minMajorVersion.get() - if (incubatorJavaVersion || manualMinMajorVersion) { - jvmArgs '--add-modules', 'jdk.incubator.vector' - if (manualMinMajorVersion) { - systemProperty 'org.apache.lucene.vectorization.upperJavaFeatureVersion', Integer.toString(minMajorVersion.get()) - } - } - - jvmArgs '--enable-native-access=' + (project.path in [ - ':lucene:core', - ':lucene:codecs', - ":lucene:distribution.tests", - ":lucene:test-framework" - ] ? 'ALL-UNNAMED' : 'org.apache.lucene.core') - - def loggingFileProvider = new LoggingFileArgumentProvider() - loggingFileProvider.setLoggingConfigFile(loggingConfigFile) - loggingFileProvider.setTempDir(tmpDirOption.get()) - jvmArgumentProviders.add(loggingFileProvider) - - systemProperty 'java.awt.headless', 'true' - systemProperty 'jdk.map.althashing.threshold', '0' - - if (!Os.isFamily(Os.FAMILY_WINDOWS)) { - systemProperty 'java.security.egd', 'file:/dev/./urandom' - } - - // Pass certain buildOptions as system properties - for (String key : optionsInheritedAsProperties) { - def option = buildOptions.optionValue(key) - if (option.isPresent()) { - systemProperty(key, buildOptions.optionValue(key).get()) - } - } - - // Turn jenkins blood red for hashmap bugs - systemProperty 'jdk.map.althashing.threshold', '0' - - // Set up cwd and temp locations. - systemProperty("java.io.tmpdir", testsTmpDir) - doFirst { - testsCwd.mkdirs() - testsTmpDir.mkdirs() - } - - jvmArgs Commandline.translateCommandline(jvmArgsOption.get()) - - // Disable HTML report generation. The reports are big and slow to generate. - reports.html.required = false - - // Set up logging. - testLogging { - events TestLogEvent.FAILED - exceptionFormat = TestExceptionFormat.FULL - showExceptions = true - showCauses = true - showStackTraces = true - stackTraceFilters.clear() - showStandardStreams = false - } - - // Disable automatic test class detection, rely on class names only. This is needed for testing - // against JDKs where the bytecode is unparseable by Gradle, for example. - // We require all tests to start with Test*, this simplifies include patterns greatly. - scanForTestClasses = false - include '**/Test*.class' - exclude '**/*$*' - - // Set up custom test output handler. - doFirst { - project.delete testOutputsDir - } - - def spillDir = getTemporaryDir().toPath() - def listener = new ErrorReportingTestListener(test.testLogging, spillDir, testOutputsDir.toPath(), verboseMode) - addTestOutputListener(listener) - addTestListener(listener) - - doFirst { - // Print some diagnostics about locations used. - logger.info("Test folders for {}: cwd={}, tmp={}", project.path, testsCwd, testsTmpDir) - } -} - - -class LoggingFileArgumentProvider implements CommandLineArgumentProvider { - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - RegularFile loggingConfigFile - - @Internal - Directory tempDir - - @Override - Iterable asArguments() { - [ - "-Djava.util.logging.config.file=${loggingConfigFile.getAsFile()}", - "-DtempDir=${tempDir.getAsFile()}" - ] - } -} - - -ext { - // Resolves test option's value. - resolvedTestOption = { propName -> - return buildOptions[propName].get() - } - - commonDir = project(":lucene").projectDir -} diff --git a/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-releases.gradle b/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-releases.gradle index 62c1a5417f78..f4204e8f7fce 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-releases.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-releases.gradle @@ -15,6 +15,7 @@ * limitations under the License. */ +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension // Configure artifact push to apache nexus (releases repository). def apacheNexusReleasesRepository = "https://repository.apache.org/service/local/staging/deploy/maven2" @@ -38,11 +39,13 @@ tasks.register("mavenToApacheReleases", { } }) +LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) + def checkReleasesRepositoryPushPreconditions = tasks.register("checkReleasesRepositoryPushPreconditions", { doFirst { // Make sure we're pushing a release version. The release repository // does not accept snapshots and returns cryptic errors upon trying. - if (rootProject.ext.snapshotBuild) { + if (buildGlobals.snapshotBuild) { throw new GradleException("ASF releases repository will not accept a snapshot version: ${rootProject.version}") } diff --git a/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-snapshots.gradle b/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-snapshots.gradle index 29e5796479d6..9fbeb5772849 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-snapshots.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.publications.maven-to-nexus-snapshots.gradle @@ -15,6 +15,7 @@ * limitations under the License. */ +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension // Configure artifact push to apache nexus (snapshots repository, CI job) def apacheNexusSnapshotsRepository = "https://repository.apache.org/content/repositories/snapshots" @@ -38,10 +39,12 @@ tasks.register("mavenToApacheSnapshots", { } }) +LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) + def checkSnapshotsRepositoryPushPreconditions = tasks.register("checkSnapshotsRepositoryPushPreconditions", { doFirst { // Make sure we're pushing a snapshot version. - if (!rootProject.ext.snapshotBuild) { + if (!buildGlobals.snapshotBuild) { throw new GradleException("ASF snapshots repository will not accept a non-snapshot version: ${rootProject.version}") } diff --git a/build-tools/build-infra/src/main/groovy/lucene.root-project.setup.gradle b/build-tools/build-infra/src/main/groovy/lucene.root-project.setup.gradle index bf63ca8d5f58..06216043aa5b 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.root-project.setup.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.root-project.setup.gradle @@ -40,10 +40,12 @@ allprojects { } plugins.apply(BasePlugin) + +plugins.apply(RegisterBuildGlobalsPlugin) + plugins.apply(GitInfoPlugin) plugins.apply(AstGrepPlugin) plugins.apply(EditorConfigLintPlugin) -plugins.apply(RegisterBuildGlobalsPlugin) // // Expose parts of the final project version to the build. We can't use Runtime.Version, sadly. @@ -51,10 +53,6 @@ plugins.apply(RegisterBuildGlobalsPlugin) def buildGlobals = rootProject.extensions.getByName("buildGlobals") ext { // TODO: remove ext properties in favor of using the global extension directly in Java plugins code (typed refs). - baseVersion = buildGlobals.baseVersion - majorVersion = buildGlobals.majorVersion - snapshotBuild = buildGlobals.snapshotBuild - isCIBuild = buildGlobals.isCIBuild // Minimum Java version required to compile and run Lucene. minJavaVersion = JavaVersion.toVersion(deps.versions.minJava.get()) @@ -75,38 +73,6 @@ ext { isIdeaBuild = (isIdea && !isIdeaSync) } -ext { - // JDK versions where the vector module is still incubating; - // also change this in extractor tool: ExtractForeignAPI - vectorIncubatorJavaVersions = [ - JavaVersion.VERSION_21, - JavaVersion.VERSION_22, - JavaVersion.VERSION_23, - JavaVersion.VERSION_24 - ] as Set -} - -// Pick the "root" seed from which everything else that is randomized is derived. -Provider rootSeedOption = buildOptions.addOption("tests.seed", "The \"root\" randomization seed for options and test parameters.", provider { - return String.format("%08X", new Random().nextLong()) -}) -ext { - rootSeed = rootSeedOption.get() - rootSeedUserProvided = buildOptions.getOption("tests.seed").getValue().get().source() == BuildOptionValueSource.COMPUTED_VALUE - rootSeedLong = SeedUtils.parseSeedChain(rootSeed)[0] -} -allprojects { - ext { - projectSeedLong = rootProject.rootSeedLong ^ project.path.hashCode() - } -} - -tasks.register("showTestsSeed", { - doFirst { - logger.lifecycle("Running tests with root randomization seed: tests.seed=${rootProject.ext.rootSeed}") - } -}) - // Wire up included builds to some validation tasks. tasks.matching { it.name == "tidy" }.configureEach { dependsOn gradle.includedBuilds*.task(":tidy") diff --git a/build-tools/build-infra/src/main/groovy/lucene.validation.error-prone.gradle b/build-tools/build-infra/src/main/groovy/lucene.validation.error-prone.gradle index 63703c7fdd7b..ea48716c2ede 100644 --- a/build-tools/build-infra/src/main/groovy/lucene.validation.error-prone.gradle +++ b/build-tools/build-infra/src/main/groovy/lucene.validation.error-prone.gradle @@ -15,21 +15,26 @@ * limitations under the License. */ +import org.apache.lucene.gradle.plugins.java.AlternativeJdkSupportPlugin +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension + // Applies error-prone for additional linting. if (project != project.rootProject) { throw new GradleException("Applicable to rootProject only: " + project.path) } +LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) + Provider errorproneOption = buildOptions.addBooleanOption("validation.errorprone", - "Applies error-prone for additional source code linting.", provider { -> rootProject.ext.isCIBuild as boolean }) + "Applies error-prone for additional source code linting.", buildGlobals.isCIBuild) // make sure we can apply error-prone or provide a reason why we can't. String skipReason = null -if (rootProject.ext.usesAltJvm) { +if (rootProject.getExtensions().getByType(AlternativeJdkSupportPlugin.AltJvmExtension).altJvmUsed.get()) { skipReason = "won't work with alternative java toolchain" } else if (errorproneOption.get() == false) { - if (rootProject.ext.isCIBuild) { + if (buildGlobals.isCIBuild) { throw new GradleException("Odd, errorprone linting should always be enabled on CI builds.") } else { skipReason = "skipped on builds not running inside CI environments, pass -Pvalidation.errorprone=true to enable" diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/LuceneGradlePlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/LuceneGradlePlugin.java index 3a04c20d16bd..32129e914aa6 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/LuceneGradlePlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/LuceneGradlePlugin.java @@ -16,6 +16,7 @@ */ package org.apache.lucene.gradle.plugins; +import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsExtension; import java.nio.file.Path; import java.util.Locale; import org.gradle.api.GradleException; @@ -62,4 +63,23 @@ protected void applicableToRootProjectOnly(Project project) { "This plugin is applicable to the rootProject only: " + getClass().getSimpleName()); } } + + /** + * Returns a filesystem path to a given resource that the plugin uses. At the moment, these + * resources are located under the top-level {@code gradle/} folder. + */ + protected Path gradlePluginResource(Project project, String relativePath) { + return project + .getLayout() + .getSettingsDirectory() + .dir("gradle") + .getAsFile() + .toPath() + .resolve(relativePath); + } + + /** Utility method returning {@link BuildOptionsExtension}. */ + protected BuildOptionsExtension getBuildOptions(Project project) { + return project.getExtensions().getByType(BuildOptionsExtension.class); + } } diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java index f4d8510beccb..103eec9ba51d 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java @@ -16,6 +16,8 @@ */ package org.apache.lucene.gradle.plugins.globals; +import org.gradle.api.provider.Property; + /** Global build constants. */ public abstract class LuceneBuildGlobalsExtension { public static final String NAME = "buildGlobals"; @@ -50,4 +52,13 @@ public abstract class LuceneBuildGlobalsExtension { * */ public boolean isCIBuild; + + /** Returns per-project seed for randomization. */ + public abstract Property getProjectSeedAsLong(); + + /** Return the root randomization seed */ + public abstract Property getRootSeed(); + + /** Return the root randomization seed as a {@code long} value. */ + public abstract Property getRootSeedAsLong(); } diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java index 9c6e35ed23fe..1788dd75bce7 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java @@ -18,8 +18,10 @@ import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsExtension; import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsPlugin; +import com.carrotsearch.randomizedtesting.SeedUtils; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Random; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; @@ -52,6 +54,18 @@ public void apply(Project project) { System.getenv().keySet().stream() .anyMatch(key -> key.matches("(?i)((JENKINS|HUDSON)(_\\w+)?|CI)")); + // Pick the "root" seed from which everything else that is randomized is derived. + Provider rootSeedOption = + getBuildOptions(project) + .addOption( + "tests.seed", + "The \"root\" randomization seed for options and test parameters.", + project.provider(() -> String.format("%08X", new Random().nextLong()))); + String rootSeed = rootSeedOption.get(); + + // We take just the root seed, ignoring any chained sub-seeds. + long rootSeedLong = SeedUtils.parseSeedChain(rootSeed)[0]; + project.allprojects( p -> { var globals = @@ -64,6 +78,12 @@ public void apply(Project project) { globals.buildTime = buildTime; globals.buildYear = buildYear; globals.isCIBuild = isCIBuild; + globals.getRootSeed().set(rootSeed); + globals.getRootSeedAsLong().set(rootSeedLong); + globals + .getProjectSeedAsLong() + .convention(rootSeedLong ^ p.getPath().hashCode()) + .finalizeValue(); }); } diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java index 3621ce9df5b7..50e2b51b5f2b 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java @@ -48,7 +48,10 @@ public void apply(Project project) { "hunspell.repo.path", "validation.owasp", "validation.owasp.apikey", - "validation.owasp.threshold")); + "validation.owasp.threshold", + "tests.linedocsfile", + "tests.LUCENE_VERSION", + "tests.bwcdir")); optionGroups.group("Test profiling", "tests\\.profile\\.(.*)"); diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/AlternativeJdkSupportPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/AlternativeJdkSupportPlugin.java new file mode 100644 index 000000000000..8405d8a28dc9 --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/AlternativeJdkSupportPlugin.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.java; + +import java.io.File; +import java.util.Locale; +import java.util.function.Function; +import javax.inject.Inject; +import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; +import org.gradle.api.JavaVersion; +import org.gradle.api.Project; +import org.gradle.api.file.Directory; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.compile.JavaCompile; +import org.gradle.api.tasks.testing.Test; +import org.gradle.internal.jvm.JavaInfo; +import org.gradle.internal.jvm.Jvm; +import org.gradle.internal.jvm.inspection.JvmInstallationMetadata; +import org.gradle.internal.jvm.inspection.JvmMetadataDetector; +import org.gradle.jvm.toolchain.internal.InstallationLocation; + +// I failed to set it up leveraging Gradle's toolchains because +// a toolchain spec is not flexible enough to provide an exact location of the JVM to be used; +// if you have two identical JVM lang. versions in auto-discovered JVMs, an arbitrary one is used +// (?). +// This situation is not uncommon when debugging low-level stuff (hand-compiled JVM binaries). + +/** This adds support for compiling and testing against a different Java runtime. */ +public abstract class AlternativeJdkSupportPlugin extends LuceneGradlePlugin { + public abstract static class AltJvmExtension { + public abstract Property getAltJvmUsed(); + + public abstract Property getCompilationJvm(); + + public abstract Property getGradleJvm(); + + public abstract Property getCompilationJvmVersion(); + } + + public abstract static class RootHooksPlugin extends LuceneGradlePlugin { + @Inject + public abstract JvmMetadataDetector getJvmMetadataDetector(); + + @Override + public void apply(Project project) { + applicableToRootProjectOnly(project); + + Provider runtimeJavaHomeOption = + getBuildOptions(project) + .addDirOption( + "runtime.java.home", + "Home directory path to an alternative compilation/ runtime JDK."); + + // we used to have "RUNTIME_JAVA_HOME" uppercase env variable support so keep this option too. + var legacyUpperCase = + project + .getLayout() + .getProjectDirectory() + .dir(project.getProviders().environmentVariable("RUNTIME_JAVA_HOME")); + runtimeJavaHomeOption = legacyUpperCase.orElse(runtimeJavaHomeOption); + + JavaInfo jvmGradle = Jvm.current(); + JavaInfo jvmCurrent = + runtimeJavaHomeOption.isPresent() + ? Jvm.forHome(runtimeJavaHomeOption.get().getAsFile()) + : jvmGradle; + + var altJvmExt = project.getExtensions().create("altJvmExtension", AltJvmExtension.class); + + boolean altJvmUsed = !jvmGradle.getJavaHome().equals(jvmCurrent.getJavaHome()); + + JvmMetadataDetector jvmDetector = getJvmMetadataDetector(); + altJvmExt.getAltJvmUsed().convention(altJvmUsed).finalizeValue(); + altJvmExt.getCompilationJvm().convention(jvmCurrent).finalizeValue(); + altJvmExt.getGradleJvm().convention(jvmGradle).finalizeValue(); + altJvmExt + .getCompilationJvmVersion() + .convention( + jvmDetector + .getMetadata( + InstallationLocation.userDefined(jvmCurrent.getJavaHome(), "specific path")) + .getLanguageVersion()) + .finalizeValue(); + + Function jvmInfo = + javaInfo -> { + JvmInstallationMetadata jvmMetadata = + jvmDetector.getMetadata( + InstallationLocation.userDefined(javaInfo.getJavaHome(), "specific path")); + return String.format( + Locale.ROOT, + "%s (%s %s, home at: %s)", + jvmMetadata.getLanguageVersion(), + jvmMetadata.getDisplayName(), + jvmMetadata.getRuntimeVersion(), + jvmMetadata.getJavaHome()); + }; + + project + .getTasks() + .register( + "altJvmWarning", + task -> { + task.doFirst( + t -> { + t.getLogger() + .warn( + "NOTE: Alternative java toolchain will be used for compilation and tests:\nProject will use {}\nGradle runs with {}", + jvmInfo.apply(altJvmExt.getCompilationJvm().get()), + jvmInfo.apply(altJvmExt.getGradleJvm().get())); + }); + }); + } + } + + @Override + public void apply(Project project) { + requiresAppliedPlugin(project, JavaPlugin.class); + + // Set up the globals on the root project first. + project.getRootProject().getPlugins().apply(RootHooksPlugin.class); + + var altJvmExt = project.getRootProject().getExtensions().getByType(AltJvmExtension.class); + if (altJvmExt.getAltJvmUsed().get()) { + // Set up custom JVM for tests + project + .getTasks() + .withType(Test.class) + .configureEach( + task -> { + task.dependsOn(":altJvmWarning"); + task.executable(altJvmExt.getCompilationJvm().get().getJavaExecutable()); + }); + + // Set up javac compilation tasks + project + .getTasks() + .withType(JavaCompile.class) + .configureEach( + task -> { + task.dependsOn(":altJvmWarning"); + task.getOptions().setFork(true); + task.getOptions() + .getForkOptions() + .setJavaHome(altJvmExt.getCompilationJvm().get().getJavaHome()); + }); + + // Set up javadoc compilation. + File javadocExecutable = altJvmExt.getCompilationJvm().get().getJavadocExecutable(); + project + .getTasks() + .withType(RenderJavadocTaskBase.class) + .configureEach( + task -> { + task.dependsOn(":altJvmWarning"); + task.getExecutable().set(javadocExecutable.toString()); + }); + } + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/ErrorReportingTestListener.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorReportingTestListener.java similarity index 99% rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/ErrorReportingTestListener.java rename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorReportingTestListener.java index a6dc738a6456..a4b1413fc9a5 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/ErrorReportingTestListener.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorReportingTestListener.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.gradle; +package org.apache.lucene.gradle.plugins.java; import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionValueSource; import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsExtension; diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/PrefixedWriter.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/PrefixedWriter.java similarity index 98% rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/PrefixedWriter.java rename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/PrefixedWriter.java index 3dc663e83329..b68cad22bcd6 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/PrefixedWriter.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/PrefixedWriter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.gradle; +package org.apache.lucene.gradle.plugins.java; import java.io.IOException; import java.io.Writer; diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocTaskBase.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocTaskBase.java new file mode 100644 index 000000000000..176b726ea6fc --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocTaskBase.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.java; + +import org.gradle.api.DefaultTask; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; +import org.gradle.internal.jvm.Jvm; + +/** A temporary stub before all of RenderJavadocTask is ported from the gradle script. */ +public abstract class RenderJavadocTaskBase extends DefaultTask { + @Optional + @Input + public abstract Property getExecutable(); + + public RenderJavadocTaskBase() { + getExecutable() + .convention( + getProject() + .getProviders() + .provider(() -> Jvm.current().getJavadocExecutable().toString())); + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/SpillWriter.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/SpillWriter.java similarity index 98% rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/SpillWriter.java rename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/SpillWriter.java index 9539bddbbfec..6aba63c63268 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/SpillWriter.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/SpillWriter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.gradle; +package org.apache.lucene.gradle.plugins.java; import java.io.IOException; import java.io.Reader; diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/StdOutTeeWriter.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/StdOutTeeWriter.java similarity index 98% rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/StdOutTeeWriter.java rename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/StdOutTeeWriter.java index 8bd2256e0911..a86f521be832 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/StdOutTeeWriter.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/StdOutTeeWriter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.gradle; +package org.apache.lucene.gradle.plugins.java; import java.io.IOException; import java.io.PrintStream; diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java new file mode 100644 index 000000000000..cf8010949564 --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.java; + +import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsExtension; +import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionsPlugin; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import java.io.File; +import java.nio.file.Path; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension; +import org.apache.tools.ant.taskdefs.condition.Os; +import org.apache.tools.ant.types.Commandline; +import org.gradle.api.GradleException; +import org.gradle.api.JavaVersion; +import org.gradle.api.Project; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.invocation.Gradle; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.compile.JavaCompile; +import org.gradle.api.tasks.testing.Test; +import org.gradle.api.tasks.testing.logging.TestExceptionFormat; +import org.gradle.api.tasks.testing.logging.TestLogEvent; +import org.gradle.process.CommandLineArgumentProvider; + +/** Sets up gradle's Test task configuration, including all kinds of randomized options */ +public class TestsAndRandomizationPlugin extends LuceneGradlePlugin { + public static class RootHooksPlugin extends LuceneGradlePlugin { + @Override + public void apply(Project project) { + applicableToRootProjectOnly(project); + + project + .getTasks() + .register( + "warnForcedLimitedParallelism", + task -> { + task.doFirst( + t -> { + t.getLogger() + .warn( + "'tests.jvm' build option forced to 1 because tests.verbose is true."); + }); + }); + + project + .getTasks() + .register( + "showTestsSeed", + task -> { + var testsSeedOption = + project + .getExtensions() + .getByType(BuildOptionsExtension.class) + .getOption("tests.seed"); + + String seedSource = + switch (testsSeedOption.getSource()) { + case GRADLE_PROPERTY -> "project property"; + case SYSTEM_PROPERTY -> "system property"; + case ENVIRONMENT_VARIABLE -> "environment variable"; + case EXPLICIT_VALUE -> "explicit value"; + case COMPUTED_VALUE -> "picked at random"; + case BUILD_OPTIONS_FILE -> BuildOptionsPlugin.BUILD_OPTIONS_FILE + " file"; + case LOCAL_BUILD_OPTIONS_FILE -> + BuildOptionsPlugin.LOCAL_BUILD_OPTIONS_FILE + " file"; + }; + + task.doFirst( + t -> { + t.getLogger() + .lifecycle( + "Running tests with root randomization seed tests.seed=" + + testsSeedOption.asStringProvider().get() + + ", source: " + + seedSource); + }); + }); + } + } + + @Override + public void apply(Project project) { + requiresAppliedPlugin(project, JavaPlugin.class); + + // Add warning task at the top level project so that we only emit it once. + project.getRootProject().getPlugins().apply(RootHooksPlugin.class); + + // Pass certain build options to the test JVM as system properties + LinkedHashSet optionsInheritedAsProperties = new LinkedHashSet<>(); + + BuildOptionsExtension buildOptions = getBuildOptions(project); + LuceneBuildGlobalsExtension buildGlobals = + project.getExtensions().getByType(LuceneBuildGlobalsExtension.class); + + // JVM options + Provider minHeapSizeOption = + buildOptions.addOption("tests.minheapsize", "Minimum heap size for test JVMs", "256m"); + Provider heapSizeOption = + buildOptions.addOption("tests.heapsize", "Heap size for test JVMs", "512m"); + + // Vectorization-related options. + boolean defaultVectorizationEnabled = + addVectorizationOptions(project, buildGlobals, buildOptions, optionsInheritedAsProperties); + + // Verbatim JVM arguments; make it also accept TEST_JVM_ARGS env. variable. + Provider jvmArgsOption = + project + .getProviders() + .environmentVariable("TEST_JVM_ARGS") + .orElse( + buildOptions.addOption( + "tests.jvmargs", + "Arguments passed to each forked test JVM.", + project.provider( + () -> { + return (buildGlobals.isCIBuild || defaultVectorizationEnabled) + ? "" + : "-XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1"; + }))); + + // cwd and tmp dir for forked JVMs. + var buildDirectory = project.getLayout().getBuildDirectory(); + Provider workDirOption = + buildOptions.addDirOption( + "tests.workDir", + "Working directory for forked test JVMs.", + buildDirectory.dir("tests-cwd")); + Provider tmpDirOption = + buildOptions.addDirOption( + "tests.tmpDir", + "Temp directory for forked test JVMs.", + buildDirectory.dir("tests-tmp")); + File testsCwd = workDirOption.get().getAsFile(); + File testsTmpDir = workDirOption.get().getAsFile(); + + // Asserts, debug output. + Provider verboseOption = + buildOptions.addBooleanOption( + "tests.verbose", + "Enables verbose test output mode (emits full test outputs immediately).", + false); + Provider haltOnFailureOption = + buildOptions.addBooleanOption( + "tests.haltonfailure", "Halt processing early on test failure.", false); + Provider failFastOption = + buildOptions.addBooleanOption("tests.failfast", "Stop the build early on failure.", false); + Provider rerunOption = + buildOptions.addBooleanOption( + "tests.rerun", + "Always rerun the test task, even if nothing has changed on input.", + true); + + // How many testing JVM forks to create + Provider jvmsOption = + buildOptions.addIntOption( + "tests.jvms", + "The number of forked test JVMs", + project + .getProviders() + .provider( + () -> { + return ((int) + Math.max( + 1, Math.min(Runtime.getRuntime().availableProcessors() / 2.0, 4.0))); + })); + + // GITHUB#13986: Allow easier configuration of the Panama Vectorization provider with newer Java + // versions + Provider upperJavaFeatureVersionOption = + buildOptions.addIntOption( + "tests.upperJavaFeatureVersion", + "Min JDK feature version to configure the Panama Vectorization provider"); + + // Test reiteration, filtering and component randomization options. + + // Propagate root seed so that it is visible and reported as an option in subprojects. + if (project != project.getRootProject()) { + buildOptions.addOption( + "tests.seed", + "The \"root\" randomization seed for options and test parameters.", + buildGlobals.getRootSeed()); + } + optionsInheritedAsProperties.add("tests.seed"); + + buildOptions.addIntOption("tests.iters", "Duplicate (re-run) each test case N times."); + optionsInheritedAsProperties.add("tests.iters"); + + buildOptions.addIntOption("tests.multiplier", "Value multiplier for randomized tests."); + optionsInheritedAsProperties.add("tests.multiplier"); + + buildOptions.addIntOption("tests.maxfailures", "Skip tests after a given number of failures."); + optionsInheritedAsProperties.add("tests.maxfailures"); + + buildOptions.addIntOption("tests.timeoutSuite", "Timeout (in millis) for an entire suite."); + optionsInheritedAsProperties.add("tests.timeoutSuite"); + + Provider assertsOption = + buildOptions.addBooleanOption( + "tests.asserts", "Enables or disables assertions mode.", true); + optionsInheritedAsProperties.add("tests.asserts"); + + buildOptions.addBooleanOption( + "tests.infostream", "Enables or disables infostream logs.", false); + optionsInheritedAsProperties.add("tests.infostream"); + + buildOptions.addBooleanOption( + "tests.leaveTemporary", "Leave temporary directories after tests complete.", false); + optionsInheritedAsProperties.add("tests.leaveTemporary"); + + buildOptions.addOption("tests.codec", "Sets the codec tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.codec"); + + buildOptions.addOption( + "tests.directory", "Sets the Directory implementation tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.directory"); + + buildOptions.addOption( + "tests.postingsformat", "Sets the postings format tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.postingsformat"); + + buildOptions.addOption( + "tests.docvaluesformat", "Sets the doc values format tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.docvaluesformat"); + + buildOptions.addOption( + "tests.locale", "Sets the default locale tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.locale"); + + buildOptions.addOption( + "tests.timezone", "Sets the default time zone tests should run with.", "random"); + optionsInheritedAsProperties.add("tests.timezone"); + + buildOptions.addOption("tests.filter", "Applies a test filter (see ./gradlew :helpTests)."); + optionsInheritedAsProperties.add("tests.filter"); + + buildOptions.addBooleanOption("tests.nightly", "Enables or disables @Nightly tests.", false); + buildOptions.addBooleanOption("tests.monster", "Enables or disables @Monster tests.", false); + buildOptions.addBooleanOption( + "tests.awaitsfix", "Enables or disables @AwaitsFix tests.", false); + optionsInheritedAsProperties.addAll( + List.of("tests.nightly", "tests.monster", "tests.awaitsfix")); + + buildOptions.addBooleanOption( + "tests.gui", + "Enables or disables @RequiresGUI tests.", + project.getProviders().provider(() -> buildGlobals.isCIBuild)); + + buildOptions.addOption( + "tests.file.encoding", + "Sets the default file.encoding on test JVM.", + project + .getProviders() + .provider( + () -> { + return RandomPicks.randomFrom( + new Random(buildGlobals.getProjectSeedAsLong().get()), + List.of("US-ASCII", "ISO-8859-1", "UTF-8")); + })); + + buildOptions.addBooleanOption( + "tests.faiss.run", "Explicitly run tests for the Faiss codec.", false); + optionsInheritedAsProperties.add("tests.faiss.run"); + + buildOptions.addOption( + "tests.linedocsfile", "Line docs test data file path.", "europarl.lines.txt.gz"); + optionsInheritedAsProperties.add("tests.linedocsfile"); + + buildOptions.addOption( + "tests.LUCENE_VERSION", "Base Lucene version for tests.", buildGlobals.baseVersion); + optionsInheritedAsProperties.add("tests.LUCENE_VERSION"); + + buildOptions.addOption("tests.bwcdir", "Test data for backward-compatibility indexes."); + optionsInheritedAsProperties.add("tests.bwcdir"); + + // If we're running in verbose mode and: + // 1) worker count > 1 + // 2) number of 'test' tasks in the build is > 1 + // then the output would very likely be mangled on the + // console. Fail and let the user know what to do. + boolean verboseMode = verboseOption.get(); + if (verboseMode) { + Gradle gradle = project.getGradle(); + if (gradle.getStartParameter().getMaxWorkerCount() > 1) { + gradle + .getTaskGraph() + .whenReady( + graph -> { + var testTasks = + graph.getAllTasks().stream().filter(task -> task instanceof Test).count(); + if (testTasks > 1) { + throw new GradleException( + "Run your tests in verbose mode only with --max-workers=1 option passed to gradle."); + } + }); + } + } + + Provider minMajorVersion = + upperJavaFeatureVersionOption.map( + ver -> Integer.parseInt(JavaVersion.toVersion(ver).getMajorVersion())); + var altJvmExt = + project + .getRootProject() + .getExtensions() + .getByType(AlternativeJdkSupportPlugin.AltJvmExtension.class); + JavaVersion runtimeJava = altJvmExt.getCompilationJvmVersion().get(); + + // JDK versions where the vector module is still incubating. + boolean incubatorJavaVersion = + Set.of("21", "22", "23", "24", "25").contains(runtimeJava.getMajorVersion()); + + // if the vector module is in incubator, pass lint flags to suppress excessive warnings. + if (incubatorJavaVersion) { + project + .getTasks() + .withType(JavaCompile.class) + .configureEach( + task -> { + task.getOptions().getCompilerArgs().add("-Xlint:-incubating"); + }); + } + + project + .getTasks() + .withType(Test.class) + .configureEach( + task -> { + // Running any test task should first display the root randomization seed. + task.dependsOn(":showTestsSeed"); + + File testOutputsDir = + task.getReports() + .getJunitXml() + .getOutputLocation() + .dir("outputs") + .get() + .getAsFile(); + + task.getExtensions() + .getByType(ExtraPropertiesExtension.class) + .set("testOutputsDir", testOutputsDir); + + // LUCENE-9660: Make it possible to always rerun tests, even if they're incrementally + // up-to-date. + if (rerunOption.get()) { + task.getOutputs().upToDateWhen(_ -> false); + } + + int maxParallelForks = jvmsOption.get(); + if (verboseMode && maxParallelForks != 1) { + task.dependsOn(":warnForcedLimitedParallelism"); + maxParallelForks = 1; + } + task.setMaxParallelForks(maxParallelForks); + + if (failFastOption.get()) { + task.setFailFast(true); + } + + task.setWorkingDir(testsCwd); + task.useJUnit(); + + task.setMinHeapSize(minHeapSizeOption.get()); + task.setMaxHeapSize(heapSizeOption.get()); + + task.setIgnoreFailures(!haltOnFailureOption.get()); + + if (assertsOption.get()) { + task.jvmArgs("-ea", "-esa"); + } else { + task.setEnableAssertions(false); + } + + // Lucene needs to optional modules at runtime, which we want to enforce for testing + // (if the runner JVM does not support them, it will fail tests): + task.jvmArgs("--add-modules", "jdk.management"); + + // dump heap on OOM. + task.jvmArgs("-XX:+HeapDumpOnOutOfMemoryError"); + + // Enable the vector incubator module on supported Java versions: + boolean manualMinMajorVersion = + minMajorVersion.isPresent() + && Integer.parseInt(runtimeJava.getMajorVersion()) <= minMajorVersion.get(); + if (incubatorJavaVersion || manualMinMajorVersion) { + task.jvmArgs("--add-modules", "jdk.incubator.vector"); + if (manualMinMajorVersion) { + task.systemProperty( + "org.apache.lucene.vectorization.upperJavaFeatureVersion", + Integer.toString(minMajorVersion.get())); + } + } + + task.jvmArgs( + "--enable-native-access=" + + switch (project.getPath()) { + case ":lucene:codecs", + ":lucene:core", + ":lucene:distribution.tests", + ":lucene:test-framework" -> + "ALL-UNNAMED"; + default -> "org.apache.lucene.core"; + }); + + var loggingFileProvider = + project.getObjects().newInstance(LoggingFileArgumentProvider.class); + Path loggingConfigFile = + super.gradlePluginResource(project, "testing/logging.properties"); + loggingFileProvider.getLoggingConfigFile().set(loggingConfigFile.toFile()); + loggingFileProvider.getTempDir().set(tmpDirOption.get()); + task.getJvmArgumentProviders().add(loggingFileProvider); + + task.systemProperty("java.awt.headless", "true"); + task.systemProperty("jdk.map.althashing.threshold", "0"); + + if (!Os.isFamily(Os.FAMILY_WINDOWS)) { + task.systemProperty("java.security.egd", "file:/dev/./urandom"); + } + + // Turn jenkins blood red for hashmap bugs + task.systemProperty("jdk.map.althashing.threshold", "0"); + + // Pass certain buildOptions as system properties + var sysProps = task.getSystemProperties().keySet(); + for (String key : optionsInheritedAsProperties) { + Provider option = buildOptions.optionValue(key); + if (option.isPresent() && !sysProps.contains(key)) { + task.systemProperty(key, buildOptions.optionValue(key).get()); + } + } + + // Set up cwd and temp locations. + task.systemProperty("java.io.tmpdir", testsTmpDir); + + task.doFirst( + _ -> { + testsCwd.mkdirs(); + testsTmpDir.mkdirs(); + }); + + task.jvmArgs((Object[]) Commandline.translateCommandline(jvmArgsOption.get())); + + // Disable HTML report generation. The reports are big and slow to generate. + task.getReports().getHtml().getRequired().set(false); + + // Set up logging. + var logging = task.getTestLogging(); + logging.events(TestLogEvent.FAILED); + logging.setExceptionFormat(TestExceptionFormat.FULL); + logging.setShowExceptions(true); + logging.setShowCauses(true); + logging.setShowStackTraces(true); + logging.getStackTraceFilters().clear(); + logging.setShowStandardStreams(false); + + // Disable automatic test class detection, rely on class names only. This is needed + // for testing + // against JDKs where the bytecode is unparseable by Gradle, for example. + // We require all tests to start with Test*, this simplifies include patterns greatly. + task.setScanForTestClasses(false); + task.include("**/Test*.class"); + task.exclude("**/*$*"); + + // Set up custom test output handler. + task.doFirst( + _ -> { + project.delete(testOutputsDir); + }); + + var spillDir = task.getTemporaryDir().toPath(); + var listener = + new ErrorReportingTestListener( + task.getTestLogging(), spillDir, testOutputsDir.toPath(), verboseMode); + task.addTestOutputListener(listener); + task.addTestListener(listener); + + task.doFirst( + _ -> { + task.getLogger() + .info( + "Test folders for {}: cwd={}, tmp={}", + task.getPath(), + testsCwd, + testsTmpDir); + }); + }); + } + + public abstract static class LoggingFileArgumentProvider implements CommandLineArgumentProvider { + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + public abstract RegularFileProperty getLoggingConfigFile(); + + @Internal + public abstract DirectoryProperty getTempDir(); + + @Override + public Iterable asArguments() { + return List.of( + "-Djava.util.logging.config.file=" + + getLoggingConfigFile().getAsFile().get().getAbsolutePath(), + "-DtempDir=" + getTempDir().get().getAsFile().getAbsolutePath()); + } + } + + private static boolean addVectorizationOptions( + Project project, + LuceneBuildGlobalsExtension buildGlobals, + BuildOptionsExtension buildOptions, + LinkedHashSet optionsInheritedAsProperties) { + String randomVectorSize = + RandomPicks.randomFrom( + new Random(buildGlobals.getProjectSeedAsLong().get()), + List.of("default", "128", "256", "512")); + + Provider defaultVectorizationOption = + buildOptions.addBooleanOption( + "tests.defaultvectorization", + "Uses defaults for running tests with correct JVM settings to test Panama vectorization (tests.jvmargs, tests.vectorsize, tests.forceintegervectors).", + false); + buildOptions.addOption( + "tests.vectorsize", + "Sets preferred vector size in bits.", + project.provider(() -> defaultVectorizationOption.get() ? "default" : randomVectorSize)); + + buildOptions.addBooleanOption( + "tests.forceintegervectors", + "Forces use of integer vectors even when slow.", + project.provider( + () -> defaultVectorizationOption.get() ? false : (randomVectorSize != "default"))); + + optionsInheritedAsProperties.addAll(List.of("tests.vectorsize", "tests.forceintegervectors")); + + return defaultVectorizationOption.get(); + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsBeastingPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsBeastingPlugin.java new file mode 100644 index 000000000000..3708c9a9c035 --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsBeastingPlugin.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.java; + +import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOption; +import com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionValueSource; +import java.util.List; +import java.util.Random; +import java.util.stream.IntStream; +import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.api.internal.tasks.testing.filter.DefaultTestFilter; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; +import org.gradle.api.tasks.testing.Test; + +/** + * Adds the {@code beast} task, which re-runs tests with a constant or varying root randomization + * seed. The number of re-runs is passed with a build option {@code tests.dups}. + * + *

Normal {@code --tests} test filtering options apply to restrict retries to a single test. + */ +public class TestsBeastingPlugin extends LuceneGradlePlugin { + /* + * TODO: subtasks are not run in parallel (sigh, gradle removed this capability for intra-project tasks). + * TODO: maybe it would be better to take a deeper approach and just feed the task + * runner duplicated suite names (much like https://github.com/gradle/test-retry-gradle-plugin) + * TODO: this is a somewhat related issue: https://github.com/gradle/test-retry-gradle-plugin/issues/29 + */ + + public static class RootHooksPlugin extends LuceneGradlePlugin { + @Override + public void apply(Project rootProject) { + applicableToRootProjectOnly(rootProject); + + BuildOption testsSeedOption = getBuildOptions(rootProject).getOption("tests.seed"); + + rootProject + .getTasks() + .register( + "warnAboutConstantSeed", + task -> { + task.doFirst( + t -> { + t.getLogger() + .warn( + "Root randomization seed is externally provided ({}), all duplicated runs will use the same starting seed.", + testsSeedOption.getSource().name()); + }); + }); + } + } + + @Override + public void apply(Project project) { + requiresAppliedPlugin(project, JavaPlugin.class); + + project.getRootProject().getPlugins().apply(RootHooksPlugin.class); + + BuildOption testsSeedOption = getBuildOptions(project).getOption("tests.seed"); + boolean rootSeedUserProvided = + testsSeedOption.getSource() != BuildOptionValueSource.COMPUTED_VALUE; + + var buildOptions = getBuildOptions(project); + Provider dupsOption = + buildOptions.addIntOption( + "tests.dups", "Reiterate runs of entire test suites this many times ('beast' task)."); + + boolean beastingMode = + project.getGradle().getStartParameter().getTaskNames().stream() + .anyMatch(name -> name.equals("beast") || name.endsWith(":beast")); + + var beastTask = + project + .getTasks() + .register( + "beast", + BeastTask.class, + task -> { + task.setDescription( + "Run a test suite (or a set of tests) many times over (duplicate 'test' task)."); + task.setGroup("Verification"); + }); + + if (!beastingMode) { + return; + } + + if (!dupsOption.isPresent()) { + throw new GradleException("Specify -Ptests.dups=[count] for the beast task."); + } + + // generate N test tasks and attach them to the beasting task for this + // project; + // the test filter will be applied by the beast task once it is received + // from + // command line. + long rootSeed = + project + .getExtensions() + .getByType(LuceneBuildGlobalsExtension.class) + .getRootSeedAsLong() + .get(); + + var subtasks = + IntStream.rangeClosed(1, dupsOption.get()) + .mapToObj( + idx -> { + return project + .getTasks() + .register( + "test_" + idx, + Test.class, + test -> { + test.setFailFast(true); + // If there is a user-provided root seed, use it + // (all + // duplicated tasks will run + // from the same starting seed). Otherwise, pick a + // sequential derivative. + if (!rootSeedUserProvided) { + test.systemProperty( + "tests.seed", + String.format( + "%08X", + new Random(rootSeed + (idx * 6364136223846793005L)) + .nextLong())); + } + }); + }) + .toList(); + + beastTask.configure(task -> task.dependsOn(subtasks)); + } + + /** + * We have to declare a dummy task here to be able to reuse the same syntax for 'test' task filter + * option. + */ + public abstract static class BeastTask extends DefaultTask { + @Option( + option = "tests", + description = "Sets test class or method name to be included, '*' is supported.") + public void setTestNamePatterns(List patterns) { + getTaskDependencies() + .getDependencies(this) + .forEach( + subtask -> { + if (subtask instanceof Test testTask) { + ((DefaultTestFilter) testTask.getFilter()) + .setCommandLineIncludePatterns(patterns); + } + }); + } + + @TaskAction + void run() {} + } +} diff --git a/build.gradle b/build.gradle index e70d7cc21151..7d8d413d58e1 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,6 @@ plugins { id "lucene.help" - id "lucene.java.alternative-jdk-support" id "lucene.java-projects.conventions" id "lucene.java.core.mrjar" @@ -35,7 +34,6 @@ plugins { id "lucene.java.slowest-tests-at-end" id "lucene.java.show-failed-tests-at-end" id "lucene.java.profiling" - id "lucene.java.beasting" id "lucene.java.coverage" id "lucene.java.modules" diff --git a/lucene/luke/build.gradle b/lucene/luke/build.gradle index b85ece891d30..b90b8eeda7b4 100644 --- a/lucene/luke/build.gradle +++ b/lucene/luke/build.gradle @@ -14,6 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import org.apache.lucene.gradle.plugins.java.AlternativeJdkSupportPlugin import org.apache.tools.ant.filters.* description = 'Luke - Lucene Toolbox' @@ -141,7 +143,7 @@ tasks.register("run", { doFirst { logger.lifecycle("Launching Luke ${project.version} right now...") ant.exec( - executable: rootProject.ext.runtimeJavaExecutable, + executable: rootProject.getExtensions().getByType(AlternativeJdkSupportPlugin.AltJvmExtension).compilationJvm.get().javaExecutable.toString(), spawn: true, vmlauncher: true ) {