build.gradle

// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import org.ajoberstar.grgit.Grgit

buildscript {
  repositories {
    mavenCentral()
  }
  apply from: file('gradle/buildscript.gradle'), to: buildscript

  dependencies {
    // For Apache Rat plugin to ignore non-Git files
    classpath "org.ajoberstar:grgit:1.5.0"
  }
}

allprojects {
  apply plugin: 'idea'
  apply plugin: 'eclipse'
  repositories {
    mavenCentral()
  }
}

ext {
  gradleVersion = "2.11"
  buildVersionFileName = "kafka-version.properties"

  userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null

  skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
  shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && project.gradle.startParameter.taskNames.any { it.contains("upload") }

  mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
  mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : ''
  mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : ''

  userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null

}

apply from: "$rootDir/gradle/dependencies.gradle"
apply from: file('wrapper.gradle')

if (new File('.git').exists()) {
  apply from: file('gradle/rat.gradle')
  rat {
    // Exclude everything under the directory that git should be ignoring via .gitignore or that isn't checked in. These
    // restrict us only to files that are checked in or are staged.
    def repo = Grgit.open(project.file('.'))
    excludes = new ArrayList<String>(repo.clean(ignore: false, directories: true, dryRun: true))
    // And some of the files that we have checked in should also be excluded from this check
    excludes.addAll([
        '**/.git/**',
        '**/build/**',
        'CONTRIBUTING.md',
        'gradlew',
        'gradlew.bat',
        '**/README.md'
    ])
  }
}

subprojects {
  apply plugin: 'java'
  apply plugin: 'maven'
  apply plugin: 'signing'
  apply plugin: 'checkstyle'

  sourceCompatibility = 1.7

  if (JavaVersion.current().isJava8Compatible()) {
    tasks.withType(Javadoc) {
        // disable the crazy super-strict doclint tool in Java 8
        //noinspection SpellCheckingInspection
        options.addStringOption('Xdoclint:none', '-quiet')
      }
  }

  uploadArchives {
    repositories {
      signing {
          required { shouldSign }
          sign configurations.archives

          // To test locally, replace mavenUrl in ~/.gradle/gradle.properties to file://localhost/tmp/myRepo/
          mavenDeployer {
              beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
              repository(url: "${mavenUrl}") {
                  authentication(userName: "${mavenUsername}", password: "${mavenPassword}")
              }
              afterEvaluate {
                  pom.artifactId = "${archivesBaseName}"
                  pom.project {
                      name 'Apache Kafka'
                      packaging 'jar'
                      url 'http://kafka.apache.org'
                      licenses {
                          license {
                              name 'The Apache Software License, Version 2.0'
                              url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
                              distribution 'repo'
                          }
                      }
                  }
              }
          }
      }
    }
  }

  test {
    maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
    testLogging {
      events "passed", "skipped", "failed"
      showStandardStreams = userShowStandardStreams ?: false
      exceptionFormat = 'full'
    }
  }

  jar {
    from '../LICENSE'
    from '../NOTICE'
  }

  task srcJar(type: Jar) {
    classifier = 'sources'
    from '../LICENSE'
    from '../NOTICE'
    from sourceSets.main.allSource
  }

  task javadocJar(type: Jar, dependsOn: javadoc) {
    classifier 'javadoc'
    from '../LICENSE'
    from '../NOTICE'
    from javadoc.destinationDir
  }

  task docsJar(dependsOn: javadocJar)

  task systemTestLibs(dependsOn: jar)

  artifacts {
    archives srcJar
    archives javadocJar
  }

  if(!sourceSets.test.allSource.isEmpty()) {
    task testJar(type: Jar) {
      classifier = 'test'
      from '../LICENSE'
      from '../NOTICE'
      from sourceSets.test.output
    }

    task testSrcJar(type: Jar, dependsOn: testJar) {
      classifier = 'test-sources'
      from '../LICENSE'
      from '../NOTICE'
      from sourceSets.test.allSource
    }

    artifacts {
      archives testJar
      archives testSrcJar
    }
  }

  plugins.withType(ScalaPlugin) {
    task scaladocJar(type:Jar) {
      classifier = 'scaladoc'
      from '../LICENSE'
      from '../NOTICE'
      from scaladoc.destinationDir
    }

    //documentation task should also trigger building scala doc jar
    docsJar.dependsOn scaladocJar

    artifacts {
      archives scaladocJar
    }
  }

  tasks.withType(ScalaCompile) {
    scalaCompileOptions.useAnt = false

    configure(scalaCompileOptions.forkOptions) {
      memoryMaximumSize = '1g'
      jvmArgs = ['-XX:MaxPermSize=512m', '-Xss2m']
    }
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
    configProperties = [importControlFile: "$rootDir/checkstyle/import-control.xml"]
  }
  test.dependsOn('checkstyleMain', 'checkstyleTest')
}

for ( sv in ['2_10', '2_11'] ) {
  String svInDot = sv.replaceAll( "_", ".")

  tasks.create(name: "jar_core_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['core:jar']
  }

  tasks.create(name: "test_core_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['core:test']
  }

  tasks.create(name: "srcJar_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['core:srcJar']
  }

  tasks.create(name: "docsJar_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['core:docsJar']
  }

  tasks.create(name: "install_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['install']
  }

  tasks.create(name: "releaseTarGz_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['releaseTarGz']
  }

  tasks.create(name: "uploadCoreArchives_${sv}", type: GradleBuild) {
    startParameter = project.getGradle().getStartParameter().newInstance()
    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
    tasks = ['core:uploadArchives']
  }
}

def connectPkgs = ['connect:api', 'connect:runtime', 'connect:json', 'connect:file']
def pkgs = ['clients', 'examples', 'log4j-appender', 'tools', 'streams', 'streams:examples'] + connectPkgs

tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {}
tasks.create(name: "jarAll", dependsOn: ['jar_core_2_10', 'jar_core_2_11'] + pkgs.collect { it + ":jar" }) { }

tasks.create(name: "srcJarAll", dependsOn: ['srcJar_2_10', 'srcJar_2_11'] + pkgs.collect { it + ":srcJar" }) { }

tasks.create(name: "docsJarAll", dependsOn: ['docsJar_2_10', 'docsJar_2_11'] + pkgs.collect { it + ":docsJar" }) { }

tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {}
tasks.create(name: "testAll", dependsOn: ['test_core_2_10', 'test_core_2_11'] + pkgs.collect { it + ":test" }) { }

tasks.create(name: "installAll", dependsOn: ['install_2_10', 'install_2_11'] + pkgs.collect { it + ":install" }) { }

tasks.create(name: "releaseTarGzAll", dependsOn: ['releaseTarGz_2_10', 'releaseTarGz_2_11']) { }

tasks.create(name: "uploadArchivesAll", dependsOn: ['uploadCoreArchives_2_10', 'uploadCoreArchives_2_11'] + pkgs.collect { it + ":uploadArchives" }) { }

project(':core') {
  println "Building project 'core' with Scala version ${versions.scala}"

  apply plugin: 'scala'
  archivesBaseName = "kafka_${versions.baseScala}"

  dependencies {
    compile project(':clients')
    compile libs.joptSimple
    compile libs.metrics
    compile libs.scala
    compile libs.slf4jlog4j
    compile libs.zkclient
    compile libs.zookeeper
    // These modules were broken out of core scala in 2.10. We can remove special handling when 2.10 support is dropped.
    if (versions.baseScala != '2.10') {
      compile libs.scalaParserCombinators
    }

    testCompile project(':clients').sourceSets.test.output
    testCompile libs.bcpkix
    testCompile libs.easymock
    testCompile libs.hadoopMiniKdc
    testCompile libs.junit
    testCompile libs.scalaTest
  }

  configurations {
    // manually excludes some unnecessary dependencies
    compile.exclude module: 'javax'
    compile.exclude module: 'jline'
    compile.exclude module: 'jms'
    compile.exclude module: 'jmxri'
    compile.exclude module: 'jmxtools'
    compile.exclude module: 'mail'
    compile.exclude module: 'netty'
    // To prevent a UniqueResourceException due the same resource existing in both
    // org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data
    testCompile.exclude module: 'api-ldap-schema-data'
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('slf4j-log4j12*')
    }
    from (configurations.runtime) {
      exclude('kafka-clients*')
    }
    into "$buildDir/dependant-libs-${versions.scala}"
  }

  tasks.create(name: "genProducerConfigDocs", dependsOn:jar, type: JavaExec) {
    classpath = sourceSets.main.runtimeClasspath
    main = 'org.apache.kafka.clients.producer.ProducerConfig'
    standardOutput = new File("$rootDir/docs/producer_config.html").newOutputStream()
  }

  tasks.create(name: "genConsumerConfigDocs", dependsOn:jar, type: JavaExec) {
    classpath = sourceSets.main.runtimeClasspath
    main = 'org.apache.kafka.clients.consumer.ConsumerConfig'
    standardOutput = new File("$rootDir/docs/consumer_config.html").newOutputStream()
  }

  tasks.create(name: "genKafkaConfigDocs", dependsOn:jar, type: JavaExec) {
    classpath = sourceSets.main.runtimeClasspath
    main = 'kafka.server.KafkaConfig'
    standardOutput = new File("$rootDir/docs/kafka_config.html").newOutputStream()
  }

  task siteDocsTar(dependsOn: ['genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', ':connect:runtime:genConnectConfigDocs'], type: Tar) {
    classifier = 'site-docs'
    compression = Compression.GZIP
    from project.file("../docs")
    into 'site-docs'
  }

  tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
    into "kafka_${versions.baseScala}-${version}"
    compression = Compression.GZIP
    from(project.file("../bin")) { into "bin/" }
    from(project.file("../config")) { into "config/" }
    from '../LICENSE'
    from '../NOTICE'
    from(configurations.runtime) { into("libs/") }
    from(configurations.archives.artifacts.files) { into("libs/") }
    from(project.siteDocsTar) { into("site-docs/") }
    from(project(':tools').jar) { into("libs/") }
    from(project(':tools').configurations.runtime) { into("libs/") }
    from(project(':connect:api').jar) { into("libs/") }
    from(project(':connect:api').configurations.runtime) { into("libs/") }
    from(project(':connect:runtime').jar) { into("libs/") }
    from(project(':connect:runtime').configurations.runtime) { into("libs/") }
    from(project(':connect:json').jar) { into("libs/") }
    from(project(':connect:json').configurations.runtime) { into("libs/") }
    from(project(':connect:file').jar) { into("libs/") }
    from(project(':connect:file').configurations.runtime) { into("libs/") }
    from(project(':streams').jar) { into("libs/") }
    from(project(':streams:examples').jar) { into("libs/") }
  }

  jar {
    dependsOn('copyDependantLibs', 'copyDependantTestLibs')
  }

  jar.manifest {
    attributes(
      'Version': "${version}"
    )
  }

  tasks.create(name: "copyDependantTestLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('*.jar')
    }
    into "$buildDir/dependant-testlibs"
  }

  checkstyle {
    configProperties = [importControlFile: "$rootDir/checkstyle/import-control-core.xml"]
  }
}

project(':examples') {
  archivesBaseName = "kafka-examples"

  dependencies {
    compile project(':core')
  }

  checkstyle {
    configProperties = [importControlFile: "$rootDir/checkstyle/import-control-core.xml"]
  }
}

project(':clients') {
  archivesBaseName = "kafka-clients"

  dependencies {
    compile libs.lz4
    compile libs.snappy
    compile libs.slf4jApi

    testCompile libs.bcpkix
    testCompile libs.junit

    testRuntime libs.slf4jlog4j
  }

  task determineCommitId {
    ext.commitId = "unknown"
    def takeFromHash = 16
    if (file("../.git/HEAD").exists()) {
      def headRef = file("../.git/HEAD").text
      if (headRef.contains('ref: ')) {
        headRef = headRef.replaceAll('ref: ', '').trim()
        if (file("../.git/$headRef").exists()) {
        commitId = file("../.git/$headRef").text.trim().take(takeFromHash)
        }
      } else {
        commitId = headRef.trim().take(takeFromHash)
      }
    }
    commitId
  }

  task createVersionFile(dependsOn: determineCommitId) {
    ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
    outputs.file receiptFile
    outputs.upToDateWhen { false }
    doLast {
      def data = [
        commitId: determineCommitId.commitId,
        version: version,
      ]

      receiptFile.parentFile.mkdirs()
      def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n")
      receiptFile.setText(content, "ISO-8859-1")
    }
  }

  jar {
    dependsOn createVersionFile
    from("$buildDir") {
        include "kafka/$buildVersionFileName"
    }
  }

  clean.doFirst {
    delete "$buildDir/kafka/"
  }

  javadoc {
    include "**/org/apache/kafka/clients/consumer/*"
    include "**/org/apache/kafka/clients/producer/*"
    include "**/org/apache/kafka/common/*"
    include "**/org/apache/kafka/common/errors/*"
    include "**/org/apache/kafka/common/serialization/*"
  }
}

project(':tools') {
    archivesBaseName = "kafka-tools"

    dependencies {
        compile project(':clients')
        compile project(':log4j-appender')
        compile libs.argparse4j
        compile libs.jacksonDatabind
        compile libs.slf4jlog4j

        testCompile project(':clients')
        testCompile libs.junit
    }

    javadoc {
        include "**/org/apache/kafka/tools/*"
    }

    tasks.create(name: "copyDependantLibs", type: Copy) {
        from (configurations.testRuntime) {
            include('slf4j-log4j12*')
        }
        from (configurations.runtime) {
            exclude('kafka-clients*')
        }
        into "$buildDir/dependant-libs-${versions.scala}"
    }

    jar {
        dependsOn 'copyDependantLibs'
    }
}

project(':streams') {
    archivesBaseName = "kafka-streams"

    dependencies {
        compile project(':clients')
        compile project(':connect:json')  // this dependency should be removed after we unify data API
        compile libs.slf4jlog4j
        compile libs.rocksDBJni
        compile libs.zkclient // this dependency should be removed after KIP-4
        compile libs.jacksonDatabind // this dependency should be removed after KIP-4

        testCompile project(':clients').sourceSets.test.output
        testCompile libs.junit
    }

    javadoc {
        include "**/org/apache/kafka/streams/*"
    }

    tasks.create(name: "copyDependantLibs", type: Copy) {
        from (configurations.testRuntime) {
            include('slf4j-log4j12*')
        }
        from (configurations.runtime) {
            exclude('kafka-clients*')
        }
        into "$buildDir/dependant-libs-${versions.scala}"
    }

    jar {
        dependsOn 'copyDependantLibs'
    }

    systemTestLibs {
        dependsOn testJar
    }
}

project(':streams:examples') {
  archivesBaseName = "kafka-streams-examples"

  dependencies {
    compile project(':streams')
    compile project(':connect:json')  // this dependency should be removed after we unify data API
  }

  javadoc {
    enabled = false
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.runtime) {
      exclude('kafka-streams*')
    }
    into "$buildDir/dependant-libs-${versions.scala}"
  }

  jar {
    dependsOn 'copyDependantLibs'
  }
}

project(':log4j-appender') {
  archivesBaseName = "kafka-log4j-appender"

  dependencies {
    compile project(':clients')
    compile libs.slf4jlog4j

    testCompile project(':clients').sourceSets.test.output
    testCompile libs.junit
  }

  javadoc {
    include "**/org/apache/kafka/log4jappender/*"
  }
}

project(':connect:api') {
  archivesBaseName = "connect-api"

  dependencies {
    compile project(':clients')
    compile libs.slf4jApi

    testCompile libs.junit

    testRuntime libs.slf4jlog4j
  }

  javadoc {
    options.links "http://docs.oracle.com/javase/7/docs/api/"
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('slf4j-log4j12*')
    }
    from (configurations.runtime) {
      exclude('kafka-clients*')
      exclude('connect-*')
    }
    into "$buildDir/dependant-libs"
  }

  jar {
    dependsOn copyDependantLibs
  }
}

project(':connect:json') {
  archivesBaseName = "connect-json"

  dependencies {
    compile project(':connect:api')
    compile libs.jacksonDatabind
    compile libs.slf4jApi

    testCompile libs.easymock
    testCompile libs.junit
    testCompile libs.powermock
    testCompile libs.powermockEasymock

    testRuntime libs.slf4jlog4j
  }

  javadoc {
    enabled = false
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('slf4j-log4j12*')
    }
    from (configurations.runtime) {
      exclude('kafka-clients*')
      exclude('connect-*')
    }
    into "$buildDir/dependant-libs"
  }

  jar {
    dependsOn copyDependantLibs
  }
}

project(':connect:runtime') {
  archivesBaseName = "connect-runtime"

  dependencies {
    compile project(':connect:api')
    compile project(':clients')
    compile project(':tools')
    compile libs.slf4jApi

    compile libs.jacksonJaxrsJsonProvider
    compile libs.jerseyContainerServlet
    compile libs.jettyServer
    compile libs.jettyServlet
    compile libs.reflections

    testCompile project(':clients').sourceSets.test.output
    testCompile libs.easymock
    testCompile libs.junit
    testCompile libs.powermock
    testCompile libs.powermockEasymock

    testRuntime project(":connect:json")
    testRuntime libs.slf4jlog4j
  }

  javadoc {
    enabled = false
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('slf4j-log4j12*')
    }
    from (configurations.runtime) {
      exclude('kafka-clients*')
      exclude('connect-*')
    }
    into "$buildDir/dependant-libs"
  }

  jar {
    dependsOn copyDependantLibs
  }

  tasks.create(name: "genConnectConfigDocs", dependsOn:jar, type: JavaExec) {
    classpath = sourceSets.main.runtimeClasspath
    main = 'org.apache.kafka.connect.runtime.distributed.DistributedConfig'
    standardOutput = new File("$rootDir/docs/connect_config.html").newOutputStream()
  }
}

project(':connect:file') {
  archivesBaseName = "connect-file"

  dependencies {
    compile project(':connect:api')
    compile libs.slf4jApi

    testCompile libs.easymock
    testCompile libs.junit
    testCompile libs.powermock
    testCompile libs.powermockEasymock

    testRuntime libs.slf4jlog4j
  }

  javadoc {
    enabled = false
  }

  tasks.create(name: "copyDependantLibs", type: Copy) {
    from (configurations.testRuntime) {
      include('slf4j-log4j12*')
    }
    from (configurations.runtime) {
      exclude('kafka-clients*')
      exclude('connect-*')
    }
    into "$buildDir/dependant-libs"
  }

  jar {
    dependsOn copyDependantLibs
  }
}