Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make GHA workflow script execution able to run in parallel #2092

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ plugins {
}

dependencies {
implementation(libs.workflows.kotlin.compilerEmbeddable)
compileOnly(libs.workflows.kotlin.compilerEmbeddable)
}

gradlePlugin {
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package org.spockframework.gradle

import groovy.transform.CompileStatic
import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.ProjectLayout
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.tasks.*
import org.gradle.workers.WorkerExecutor

import javax.inject.Inject

@CompileStatic
@UntrackedTask(because = 'imported files can import other files so inputs are not determinable upfront')
abstract class DetermineImportedFiles extends DefaultTask {
@InputFile
abstract RegularFileProperty getMainKtsFile()

@InputFiles
abstract ConfigurableFileCollection getKotlinCompilerEmbeddableClasspath()

@OutputFile
abstract RegularFileProperty getImportedFiles()

@Inject
abstract WorkerExecutor getWorkerExecutor()

@Inject
abstract ProjectLayout getLayout()

@TaskAction
def determineImportedFiles() {
workerExecutor.classLoaderIsolation {
it.classpath.from(kotlinCompilerEmbeddableClasspath)
}.submit(DetermineImportedFilesWorkAction) {
it.projectDirectory.set(layout.projectDirectory)
it.mainKtsFile.set(mainKtsFile)
it.importedFiles.set(importedFiles)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package org.spockframework.gradle

import groovy.transform.CompileStatic
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.workers.WorkAction
import org.gradle.workers.WorkParameters
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
import org.jetbrains.kotlin.psi.KtStringTemplateExpression

import static org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES
import static org.jetbrains.kotlin.config.CommonConfigurationKeys.MESSAGE_COLLECTOR_KEY

@CompileStatic
abstract class DetermineImportedFilesWorkAction implements WorkAction<Parameters> {
@Override
void execute() {
def projectDirectory = parameters.projectDirectory.get().asFile
parameters
.mainKtsFile
.get()
.asFile
.with { getImportedFiles(it) }
.collect { projectDirectory.relativePath(it).toString().replace('\\', '/') }
.unique()
.sort()
.join('\n')
.tap { parameters.importedFiles.get().asFile.text = it }
}

private List<File> getImportedFiles(File workflowScript) {
if (!workflowScript.file) {
return []
}

return PsiManager
.getInstance(
KotlinCoreEnvironment
.createForProduction(
Disposer.newDisposable(),
new CompilerConfiguration().tap {
it.put(MESSAGE_COLLECTOR_KEY, [email protected])
},
JVM_CONFIG_FILES
)
.project
)
.findFile(
new CoreLocalVirtualFile(
new CoreLocalFileSystem(),
workflowScript.toPath()
)
)
.with { it as KtFile }
.fileAnnotationList
?.annotationEntries
?.findAll { it.shortName?.asString() == 'Import' }
*.valueArgumentList
?.collectMany { it?.arguments ?: [] }
*.argumentExpression
?.findAll { it instanceof KtStringTemplateExpression }
?.collect { it as KtStringTemplateExpression }
*.entries
*.first()
?.findAll { it instanceof KtLiteralStringTemplateEntry }
?.collect { it as KtLiteralStringTemplateEntry }
?.collect { new File(workflowScript.parentFile, it.text) }
?.collectMany { getImportedFiles(it) + it }
?: []
}

static interface Parameters extends WorkParameters {
DirectoryProperty getProjectDirectory()

RegularFileProperty getMainKtsFile()

RegularFileProperty getImportedFiles()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package org.spockframework.gradle

import groovy.transform.CompileStatic
import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.*
import org.gradle.jvm.toolchain.JavaLauncher
import org.gradle.workers.WorkerExecutor

import javax.inject.Inject

@CompileStatic
abstract class PreprocessGithubWorkflow extends DefaultTask {
@InputFile
abstract RegularFileProperty getWorkflowScript()

@InputFiles
abstract ConfigurableFileCollection getImportedFiles()

@InputFiles
abstract ConfigurableFileCollection getKotlinCompilerClasspath()

@InputFiles
abstract ConfigurableFileCollection getMainKtsClasspath()

@Nested
abstract Property<JavaLauncher> getJavaLauncher()

@OutputFile
Provider<File> getWorkflowFile() {
workflowScript.map {
def workflowScript = it.asFile
workflowScript.toPath().resolveSibling("${workflowScript.name - ~/\.main\.kts$/}.yaml").toFile()
}
}

@Inject
abstract WorkerExecutor getWorkerExecutor()

PreprocessGithubWorkflow() {
group = 'github workflows'
}

@TaskAction
def determineImportedFiles() {
workerExecutor.noIsolation().submit(PreprocessGithubWorkflowWorkAction) {
it.workflowScript.set(workflowScript)
it.kotlinCompilerClasspath.from(kotlinCompilerClasspath)
it.mainKtsClasspath.from(mainKtsClasspath)
it.javaExecutable.set(javaLauncher.map { it.executablePath })
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package org.spockframework.gradle

import groovy.transform.CompileStatic
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.RegularFileProperty
import org.gradle.process.ExecOperations
import org.gradle.workers.WorkAction
import org.gradle.workers.WorkParameters

import javax.inject.Inject

@CompileStatic
abstract class PreprocessGithubWorkflowWorkAction implements WorkAction<Parameters> {
@Inject
abstract ExecOperations getExecOperations()

@Override
void execute() {
// work-around for https://youtrack.jetbrains.com/issue/KT-74830
Exception lastException = null
for (i in 1..5) {
try {
execOperations.javaexec {
it.executable = parameters.javaExecutable.get().asFile.absolutePath
it.classpath(parameters.kotlinCompilerClasspath)
it.mainClass.set('org.jetbrains.kotlin.cli.jvm.K2JVMCompiler')
it.args('-no-stdlib', '-no-reflect')
it.args('-classpath', parameters.mainKtsClasspath.asPath)
it.args('-script', parameters.workflowScript.get().asFile.absolutePath)

// work-around for https://youtrack.jetbrains.com/issue/KT-42101
it.systemProperty('kotlin.main.kts.compiled.scripts.cache.dir', '')
}
return
} catch (Exception e) {
lastException = e
}
}
throw lastException
}

static interface Parameters extends WorkParameters {
RegularFileProperty getWorkflowScript()

ConfigurableFileCollection getKotlinCompilerClasspath()

ConfigurableFileCollection getMainKtsClasspath()

RegularFileProperty getJavaExecutable()
}
}
Original file line number Diff line number Diff line change
@@ -20,69 +20,47 @@ import groovy.transform.CompileStatic
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.artifacts.VersionCatalogsExtension
import org.gradle.api.tasks.JavaExec
import org.gradle.jvm.toolchain.JavaLanguageVersion
import org.gradle.jvm.toolchain.JavaToolchainService
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
import org.jetbrains.kotlin.psi.KtStringTemplateExpression

import static org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
import static org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES

@CompileStatic
class PreprocessWorkflowsPlugin implements Plugin<Project> {
void apply(Project project) {
def libs = project.extensions.getByType(VersionCatalogsExtension).find('libs').orElseThrow(AssertionError::new)
def kotlinCompilerEmbeddableClasspath = project.configurations.detachedConfiguration(
libs.findLibrary('workflows-kotlin-compilerEmbeddable').orElseThrow(AssertionError::new).get(),
)
def kotlinCompilerClasspath = project.configurations.detachedConfiguration(
libs.findLibrary('workflows-kotlin-compiler').orElseThrow(AssertionError::new).get(),
libs.findLibrary('workflows-kotlin-scriptingCompiler').orElseThrow(AssertionError::new).get()
)
def kotlinScriptClasspath = project.configurations.detachedConfiguration(
def mainKtsClasspath = project.configurations.detachedConfiguration(
libs.findLibrary('workflows-kotlin-mainKts').orElseThrow(AssertionError::new).get()
).tap {
it.transitive = false
}

def preprocessWorkflows = project.tasks.register('preprocessWorkflows') {
it.group = 'github actions'
it.group = 'github workflows'
}
project.file('.github/workflows').eachFileMatch(~/.*\.main\.kts$/) { workflowScript ->
def workflowName = workflowScript.name - ~/\.main\.kts$/
def pascalCasedWorkflowName = workflowName
.replaceAll(/-\w/) { String it -> it[1].toUpperCase() }
.replaceFirst(/^\w/) { String it -> it[0].toUpperCase() }
def preprocessWorkflow = project.tasks.register("preprocess${pascalCasedWorkflowName}Workflow", JavaExec) {
it.group = 'github actions'

it.inputs
.file(workflowScript)
.withPropertyName('workflowScript')
it.inputs
.files(getImportedFiles(project.file(workflowScript)))
.withPropertyName("importedFiles")
it.outputs
.file(new File(workflowScript.parent, "${workflowName}.yaml"))
.withPropertyName('workflowFile')

it.javaLauncher.set project.extensions.getByType(JavaToolchainService).launcherFor {
def determineImportedFiles = project.tasks.register("determineImportedFilesFor${pascalCasedWorkflowName}Workflow", DetermineImportedFiles) {
it.mainKtsFile.set(workflowScript)
it.importedFiles.set(project.layout.buildDirectory.file("importedFilesFor${pascalCasedWorkflowName}Workflow.txt"))
it.kotlinCompilerEmbeddableClasspath.from(kotlinCompilerEmbeddableClasspath)
}
def preprocessWorkflow = project.tasks.register("preprocess${pascalCasedWorkflowName}Workflow", PreprocessGithubWorkflow) {
it.workflowScript.set(workflowScript)
it.importedFiles.from(determineImportedFiles.flatMap { it.importedFiles }.map { it.asFile.readLines() })
it.kotlinCompilerClasspath.from(kotlinCompilerClasspath)
it.mainKtsClasspath.from(mainKtsClasspath)
it.javaLauncher.set(project.extensions.getByType(JavaToolchainService).launcherFor {
it.languageVersion.set(JavaLanguageVersion.of(17))
}
it.classpath(kotlinCompilerClasspath)
it.mainClass.set 'org.jetbrains.kotlin.cli.jvm.K2JVMCompiler'
it.args('-no-stdlib', '-no-reflect')
it.args('-classpath', kotlinScriptClasspath.asPath)
it.args('-script', workflowScript.absolutePath)

// work-around for https://youtrack.jetbrains.com/issue/KT-42101
it.systemProperty('kotlin.main.kts.compiled.scripts.cache.dir', '')
})
}
project.pluginManager.withPlugin('io.spring.nohttp') {
// iff both tasks are run, workflow files should be generated before checkstyle check
@@ -95,45 +73,4 @@ class PreprocessWorkflowsPlugin implements Plugin<Project> {
}
}
}

private List<File> getImportedFiles(File workflowScript) {
if (!workflowScript.file) {
return []
}

return PsiManager
.getInstance(
KotlinCoreEnvironment
.createForProduction(
Disposer.newDisposable(),
new CompilerConfiguration().tap {
it.put(MESSAGE_COLLECTOR_KEY, [email protected])
},
JVM_CONFIG_FILES
)
.project
)
.findFile(
new CoreLocalVirtualFile(
new CoreLocalFileSystem(),
workflowScript.toPath()
)
)
.with { it as KtFile }
.fileAnnotationList
?.annotationEntries
?.findAll { it.shortName?.asString() == "Import" }
*.valueArgumentList
?.collectMany { it?.arguments ?: [] }
*.argumentExpression
?.findAll { it instanceof KtStringTemplateExpression }
?.collect { it as KtStringTemplateExpression }
*.entries
*.first()
?.findAll { it instanceof KtLiteralStringTemplateEntry }
?.collect { it as KtLiteralStringTemplateEntry }
?.collect { new File(workflowScript.parentFile, it.text) }
?.collectMany { getImportedFiles(it) + it }
?: []
}
}