Skip to content

Commit 3d33eca

Browse files
semaphore-agent-production[bot]semaphore-agent-production[bot]
committed
Merge branch '4.0' of https://github.com/confluentinc/kafka into 4.0
2 parents 9c02072 + 02d955e commit 3d33eca

File tree

69 files changed

+1662
-89
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+1662
-89
lines changed

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* @confluentinc/kafka-eng @confluentinc/ksql

‎.semaphore/project.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
17+
# template and configurations in service.yml.
18+
# Modifications in this file will be overwritten by generated content in the nightly run.
19+
# For more information, please refer to the page:
20+
# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
21+
apiVersion: v1alpha
22+
kind: Project
23+
metadata:
24+
name: kafka
25+
description: ""
26+
spec:
27+
visibility: private
28+
repository:
29+
url: [email protected]:confluentinc/kafka.git
30+
run_on:
31+
- branches
32+
- pull_requests
33+
pipeline_file: .semaphore/semaphore.yml
34+
integration_type: github_app
35+
status:
36+
pipeline_files:
37+
- path: .semaphore/semaphore.yml
38+
level: pipeline
39+
whitelist:
40+
branches:
41+
- 2.3
42+
- 2.4
43+
- 2.5
44+
- 2.6
45+
- 2.7
46+
- 2.8
47+
- 3.0
48+
- 3.1
49+
- 3.2
50+
- 3.3
51+
- 3.4
52+
- 3.5
53+
- 3.6
54+
- 3.7
55+
- trunk
56+
- master
57+
custom_permissions: true
58+
debug_permissions:
59+
- empty
60+
- default_branch
61+
- non_default_branch
62+
- pull_request
63+
- forked_pull_request
64+
- tag
65+
attach_permissions:
66+
- default_branch
67+
- non_default_branch
68+
- pull_request
69+
- forked_pull_request
70+
- tag

‎.semaphore/semaphore.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
version: v1.0
17+
name: build-test-release
18+
agent:
19+
machine:
20+
type: s1-prod-ubuntu20-04-arm64-1
21+
fail_fast:
22+
cancel:
23+
when: "true"
24+
execution_time_limit:
25+
hours: 4
26+
queue:
27+
- when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.x'"
28+
processing: parallel
29+
global_job_config:
30+
env_vars:
31+
- name: NANO_VERSION
32+
value: "true"
33+
- name: PUBLISH
34+
value: "true"
35+
- name: ENABLE_PUBLISH_ARTIFACTS
36+
value: "false"
37+
- name: ENABLE_DOWNSTREAM_TRIGGER
38+
value: "true"
39+
- name: DOWNSTREAM_PROJECTS
40+
value: "common"
41+
prologue:
42+
commands:
43+
- echo $SEMAPHORE_WORKFLOW_ID
44+
- echo $SEMAPHORE_GIT_REPO_SLUG
45+
- checkout
46+
- sem-version java 21
47+
- sem-version go 1.16.15
48+
- git config --global url."[email protected]:".insteadOf "https://github.com/"
49+
- export SEMAPHORE_CACHE_DIR=/home/semaphore
50+
- source scripts/set_env_vars.sh
51+
- source scripts/set_downstream_branch.sh
52+
blocks:
53+
- name: Gradle Build
54+
dependencies: []
55+
task:
56+
jobs:
57+
- name: Build, Compile, Validations, Publish
58+
commands:
59+
- |
60+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]]; then
61+
. vault-setup
62+
fi
63+
- >-
64+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$NANO_VERSION" = "true" ] && [ "$RELEASE_JOB" = "false" ] && [ "$ENABLE_PUBLISH_ARTIFACTS" = "true" ]; then
65+
. ci-tools ci-update-version
66+
fi
67+
- |
68+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]]; then
69+
make compile-validate
70+
fi
71+
- |
72+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$PUBLISH" = "true" ] && [ "$SEMAPHORE_GIT_REF_TYPE" != "pull-request" ] && [ "$ENABLE_PUBLISH_ARTIFACTS" = "true" ]; then \
73+
if [[ "$RELEASE_JOB" = "false" ]]; then \
74+
. ci-tools ci-push-tag; \
75+
mavenUrl=$(vault kv get v1/ci/kv/gradle/artifactory_snapshots_settings | grep mavenUrl | cut -d "," -f 2 | cut -d "'" -f 2); \
76+
elif [[ "$SEMAPHORE_GIT_BRANCH" == *-alpha* ]]; then \
77+
mavenUrl=$(vault kv get v1/ci/kv/gradle/artifactory_preview_release_settings | grep mavenUrl | cut -d "," -f 2 | cut -d "'" -f 2); \
78+
fi; \
79+
./gradlewAll -PmavenUrl=$mavenUrl -PkeepAliveMode=session uploadArchives; \
80+
fi
81+
- |
82+
echo "PUBLISH: $PUBLISH, RELEASE_JOB: $RELEASE_JOB, SEMAPHORE_GIT_REF_TYPE: $SEMAPHORE_GIT_REF_TYPE, ENABLE_DOWNSTREAM_TRIGGER: $ENABLE_DOWNSTREAM_TRIGGER"
83+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$PUBLISH" = "true" ] && [ "$RELEASE_JOB" = "false" ] && [ "$SEMAPHORE_GIT_REF_TYPE" != "pull-request" ] && [ "$ENABLE_DOWNSTREAM_TRIGGER" = "true" ]; then
84+
for project in $DOWNSTREAM_PROJECTS; do
85+
sem-trigger -p $project -b $DOWNSTREAM_BRANCH_NAME -f .semaphore/semaphore.yml
86+
done
87+
fi
88+
- name: Tests
89+
dependencies: []
90+
task:
91+
jobs:
92+
- name: Unit tests and Integration tests
93+
commands:
94+
- './gradlew
95+
unitTest integrationTest --no-daemon --stacktrace --continue
96+
-PtestLoggingEvents=started,passed,skipped,failed -PmaxParallelForks=4
97+
-PignoreFailures=true -PmaxTestRetries=1 -PmaxTestRetryFailures=5'
98+
execution_time_limit:
99+
minutes: 120
100+
epilogue:
101+
always:
102+
commands:
103+
- shopt -s globstar && test-results publish --name Test-Suite --trim-output-to 1024 --omit-output-for-passed **/build/test-results/**/TEST-*.xml
104+
- |
105+
echo "Job creation time: $((SEMAPHORE_JOB_CREATION_TIME * 1000))"
106+
echo "Current time: $(date +%s%3N)"
107+
echo $(( ($(date +%s%3N) - $SEMAPHORE_JOB_CREATION_TIME * 1000) )) > duration.txt
108+
artifact push workflow duration.txt --destination $SEMAPHORE_JOB_INDEX-test-duration-$SEMAPHORE_WORKFLOW_ID || true
109+
after_pipeline:
110+
task:
111+
agent:
112+
machine:
113+
type: s1-prod-ubuntu20-04-arm64-0
114+
jobs:
115+
- name: Metrics
116+
commands:
117+
- emit-ci-metrics -p -a test-results
118+
- name: Publish Test Results
119+
commands:
120+
- test-results gen-pipeline-report || true
121+
- name: SonarQube
122+
commands:
123+
- checkout
124+
- sem-version java 11
125+
- emit-sonarqube-data -a test-results

‎Confluent-README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Confluent Open Source
2+
==========================
3+
This is a [Confluent](https://www.confluent.io/) Open Source fork of Apache Kafka.
4+
5+
This version includes several modifications to enhance maintainability and ease-of-use.
6+
Just like Apache Kafka, COS is distributed under the Apache 2.0 license.

‎Makefile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
#Runs the compile and checkstyle error check
17+
.PHONY: compile-validate
18+
compile-validate:
19+
./gradlew clean -PskipSigning=true publishToMavenLocal build -x test --no-daemon --stacktrace -PxmlSpotBugsReport=true 2>&1 | tee build.log
20+
@error_count=$$(grep -c -E "(ERROR|error:|\[Error\]|FAILED)" build.log); \
21+
if [ $$error_count -ne 0 ]; then \
22+
echo "Compile, checkstyle or spotbugs error found"; \
23+
grep -E "(ERROR|error:|\[Error\]|FAILED)" build.log | while read -r line; do \
24+
echo "$$line"; \
25+
done; \
26+
echo "Number of compile, checkstyle and spotbug errors: $$error_count"; \
27+
exit $$error_count; \
28+
else \
29+
echo "No errors found"; \
30+
fi
31+
32+
# Below targets are used during kafka packaging for debian.
33+
34+
.PHONY: clean
35+
clean:
36+
37+
.PHONY: distclean
38+
distclean:
39+
40+
%:
41+
$(MAKE) -f debian/Makefile $@

‎Vagrantfile‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ ec2_keypair_file = nil
4141
ec2_region = "us-east-1"
4242
ec2_az = nil # Uses set by AWS
4343
ec2_ami = "ami-29ebb519"
44-
ec2_instance_type = "m3.medium"
4544
ec2_spot_instance = ENV['SPOT_INSTANCE'] ? ENV['SPOT_INSTANCE'] == 'true' : true
4645
ec2_spot_max_price = "0.113" # On-demand price for instance type
4746
ec2_user = "ubuntu"
@@ -63,6 +62,7 @@ if File.exist?(local_config_file) then
6362
eval(File.read(local_config_file), binding, "Vagrantfile.local")
6463
end
6564

65+
ec2_instance_type = "c4.xlarge"
6666
# override any instance type set by Vagrantfile.local or above via an environment variable
6767
if ENV['INSTANCE_TYPE'] then
6868
ec2_instance_type = ENV['INSTANCE_TYPE']
@@ -161,7 +161,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
161161
node.vm.provider :aws do |aws|
162162
aws.tags = {
163163
'Name' => ec2_instance_name_prefix + "-" + Socket.gethostname + "-" + name,
164-
'JenkinsBuildUrl' => ENV['BUILD_URL']
164+
'role' => 'ce-kafka',
165+
'Owner' => 'ce-kafka',
166+
'JenkinsBuildUrl' => ENV['BUILD_URL'],
167+
'SemaphoreWorkflowUrl' => ENV['SEMAPHORE_WORKFLOW_URL'],
168+
'SemaphoreJobId' => ENV['SEMAPHORE_JOB_ID'],
169+
'cflt_environment' => 'devel',
170+
'cflt_partition' => 'onprem',
171+
'cflt_managed_by' => 'iac',
172+
'cflt_managed_id' => 'kafka',
173+
'cflt_service' => 'kafka'
165174
}
166175
end
167176
end

‎bin/connect-distributed‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
exec "$0.sh" "$@"

‎bin/connect-distributed.sh‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,37 @@ fi
2222

2323
base_dir=$(dirname $0)
2424

25+
###
26+
### Classpath additions for Confluent Platform releases (LSB-style layout)
27+
###
28+
#cd -P deals with symlink from /bin to /usr/bin
29+
java_base_dir=$( cd -P "$base_dir/../share/java" && pwd )
30+
31+
# confluent-common: required by kafka-serde-tools
32+
# kafka-serde-tools (e.g. Avro serializer): bundled with confluent-schema-registry package
33+
for library in "kafka" "confluent-common" "kafka-serde-tools" "monitoring-interceptors"; do
34+
dir="$java_base_dir/$library"
35+
if [ -d "$dir" ]; then
36+
classpath_prefix="$CLASSPATH:"
37+
if [ "x$CLASSPATH" = "x" ]; then
38+
classpath_prefix=""
39+
fi
40+
CLASSPATH="$classpath_prefix$dir/*"
41+
fi
42+
done
43+
2544
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
26-
export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
45+
LOG4J_CONFIG_NORMAL_INSTALL="/etc/kafka/connect-log4j2.yaml"
46+
LOG4J_CONFIG_ZIP_INSTALL="$base_dir/../etc/kafka/connect-log4j2.yaml"
47+
if [ -e "$LOG4J_CONFIG_NORMAL_INSTALL" ]; then # Normal install layout
48+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=${LOG4J_CONFIG_NORMAL_INSTALL}"
49+
elif [ -e "${LOG4J_CONFIG_ZIP_INSTALL}" ]; then # Simple zip file layout
50+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=${LOG4J_CONFIG_ZIP_INSTALL}"
51+
else # Fallback to normal default
52+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
53+
fi
2754
fi
55+
export KAFKA_LOG4J_OPTS
2856

2957
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
3058
export KAFKA_HEAP_OPTS="-Xms256M -Xmx2G"
@@ -42,4 +70,5 @@ case $COMMAND in
4270
;;
4371
esac
4472

73+
export CLASSPATH
4574
exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectDistributed "$@"

‎bin/connect-standalone‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
exec "$0.sh" "$@"

0 commit comments

Comments
 (0)