Update Gradle wrapper to 9.0.0 (#2438) (#2442)

* Update Gradle wrapper to 9.0.0

* Fix 9.0 incompatibilities

* Update shadow plugin

(cherry picked from commit 38e813c)

Author: breskeby

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy

@@ -839,6 +839,8 @@ class BuildPlugin implements Plugin<Project> {
 
         if (!project.path.startsWith(":qa")) {
             TaskProvider<DependencyLicensesTask> dependencyLicenses = project.tasks.register('dependencyLicenses', DependencyLicensesTask.class) {
+                licensesDir = project.file("licenses")
+                outputMarker = new File(project.getBuildDir(), "dependencyLicense")
                 dependencies = project.configurations.runtimeClasspath
                 mapping from: /hadoop-.*/, to: 'hadoop'
                 mapping from: /hive-.*/, to: 'hive'

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntFixture.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.gradle.api.GradleException
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskProvider
+import groovy.ant.AntBuilder;
 
 /**
  * A fixture for integration tests which runs in a separate process launched by Ant.

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntFixtureStop.groovy

@@ -25,7 +25,7 @@ import org.gradle.api.tasks.Internal
 
 import javax.inject.Inject
 
-class AntFixtureStop extends LoggedExec {
+abstract class AntFixtureStop extends LoggedExec {
 
     @Internal
     AntFixture fixture

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntTask.groovy

@@ -30,6 +30,7 @@ import org.gradle.api.tasks.TaskAction
 
 import javax.inject.Inject
 import java.nio.charset.Charset
+import groovy.ant.AntBuilder
 
 /**
  * A task which will run ant commands.

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/DependencyLicensesTask.java

@@ -122,7 +122,7 @@ public abstract class DependencyLicensesTask extends DefaultTask {
     /**
      * The directory to find the license and sha files in.
      */
-    private File licensesDir = new File(getProject().getProjectDir(), "licenses");
+    private File licensesDir;
 
     /**
      * A map of patterns to prefix, used to find the LICENSE and NOTICE file.
@@ -134,6 +134,13 @@ public abstract class DependencyLicensesTask extends DefaultTask {
      */
     private Set<String> ignoreShas = new HashSet<>();
 
+    /**
+     * A marker output directory to allow this task being up-to-date.
+     * The check logic is exception driven so a failed tasks will not be defined
+     * by this output but when successful we can safely mark the task as up-to-date.
+     */
+    private File outputMarker;
+
     /**
      * Add a mapping from a regex pattern for the jar name, to a prefix to find
      * the LICENSE and NOTICE file for that jar.
@@ -257,12 +264,13 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
 
     }
 
-    // This is just a marker output folder to allow this task being up-to-date.
-    // The check logic is exception driven so a failed tasks will not be defined
-    // by this output but when successful we can safely mark the task as up-to-date.
     @OutputDirectory
     public File getOutputMarker() {
-        return new File(getProject().getBuildDir(), "dependencyLicense");
+        return outputMarker;
+    }
+
+    public void setOutputMarker(File outputMarker) {
+        this.outputMarker = outputMarker;
     }
 
     private void failIfAnyMissing(String item, Boolean exists, String type) {

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/LoggedExec.java

@@ -50,7 +50,7 @@
  * A wrapper around gradle's Exec task to capture output and log on error.
  */
 @SuppressWarnings("unchecked")
-public class LoggedExec extends Exec implements FileSystemOperationsAware {
+public abstract class LoggedExec extends Exec implements FileSystemOperationsAware {
 
     private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
     private Consumer<Logger> outputLogger;

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/Util.java

@@ -25,7 +25,7 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.util.PatternFilterable;
 
@@ -134,7 +134,7 @@ public static FileTree getJavaTestAndMainSourceResources(Project project, Action
      * @return An Optional that contains the Java test SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME));
     }
@@ -144,7 +144,7 @@ public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
      * @return An Optional that contains the Java main SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaMainSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME));
     }

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/info/GlobalBuildInfoPlugin.java

@@ -27,6 +27,7 @@
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.provider.Provider;
+import org.gradle.process.ExecOperations;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
 import org.gradle.internal.jvm.inspection.JavaInstallationRegistry;
@@ -75,13 +76,16 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private final JavaInstallationRegistry javaInstallationRegistry;
     private final JvmMetadataDetector metadataDetector;
     private final ProviderFactory providers;
+    private final ExecOperations execOperations;
 
     @Inject
     public GlobalBuildInfoPlugin(
+            ExecOperations execOperations,
         JavaInstallationRegistry javaInstallationRegistry,
         JvmMetadataDetector metadataDetector,
         ProviderFactory providers
     ) {
+        this.execOperations = execOperations;
         this.javaInstallationRegistry = javaInstallationRegistry;
         this.metadataDetector = metadataDetector;
         this.providers = providers;
@@ -249,7 +253,7 @@ private File findRuntimeJavaHome() {
     }
 
     private String findJavaHome(String version) {
-        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv").forUseAtConfigurationTime();
+        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
         String javaHomeEnvVar = getJavaHomeEnvVarName(version);
 
         // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
@@ -318,15 +322,18 @@ private static int findDefaultParallel(Project project) {
                 }
                 _defaultParallel = socketToCore.values().stream().mapToInt(i -> i).sum();
             } else if (OS.current() == OS.MAC) {
-                // Ask macOS to count physical CPUs for us
-                ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-                project.exec(spec -> {
-                    spec.setExecutable("sysctl");
-                    spec.args("-n", "hw.physicalcpu");
-                    spec.setStandardOutput(stdout);
-                });
-
-                _defaultParallel = Integer.parseInt(stdout.toString().trim());
+                // On Apple silicon, we only want to use the performance cores
+                boolean isAppleSilicon = project.getProviders().systemProperty("os.arch").getOrElse("").equals("aarch64");
+                String query = isAppleSilicon && isMontereyOrNewer(project.getProviders())
+                        ? "hw.perflevel0.physicalcpu"
+                        : "hw.physicalcpu";
+
+                String stdout = project.getProviders().exec(execSpec ->
+                        execSpec.commandLine("sysctl", "-n", query)
+                ).getStandardOutput().getAsText().get();
+
+
+                _defaultParallel = Integer.parseInt(stdout.trim());
             }
 
             _defaultParallel = Runtime.getRuntime().availableProcessors() / 2;
@@ -335,6 +342,20 @@ private static int findDefaultParallel(Project project) {
         return _defaultParallel;
     }
 
+    private final static int MACOS_MONTEREY_MAJOR_VERSION = 12;
+
+    private static boolean isMontereyOrNewer(ProviderFactory providers) {
+        String rawVersion = providers.systemProperty("os.version").getOrElse("").trim();
+        if (rawVersion.isEmpty()) {
+            LOGGER.warn("Failed to validate MacOs version.");
+            return false;
+        }
+
+        String majorVersion = rawVersion.split("\\.")[0];
+
+        return Integer.parseInt(majorVersion) >= MACOS_MONTEREY_MAJOR_VERSION;
+    }
+
     public static GitInfo gitInfo(File rootDir) {
         try {
             /*

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/scala/SparkVariantPlugin.java

@@ -39,12 +39,9 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.SourceDirectorySet;
 import org.gradle.api.internal.artifacts.repositories.resolver.ComponentMetadataDetailsAdapter;
-import org.gradle.api.internal.tasks.DefaultScalaSourceSet;
 import org.gradle.api.plugins.BasePlugin;
-import org.gradle.api.plugins.Convention;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPlugin;
-import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;