From cf7bb1e2fb985d9656fd15346e041b4bd1f38d3a Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 26 Aug 2024 23:05:41 +0800 Subject: [PATCH 001/155] [type:feat]add kafka logging e2e test --- .../k8s/script/e2e-http-sync.sh | 4 + .../shenyu-e2e-case-http/k8s/shenu-kafka.yml | 98 +++++++++++++++++++ .../shenyu-e2e-case-http/pom.xml | 5 + .../e2e/testcase/http/DividePluginCases.java | 70 +++++++++++++ .../e2e/testcase/http/DividePluginTest.java | 10 ++ 5 files changed, 187 insertions(+) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh index 25f37f01d03f..d3e61c4f7e51 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh @@ -37,6 +37,8 @@ for sync in ${SYNC_ARRAY[@]}; do kubectl apply -f "${PRGDIR}"/shenyu-rocketmq.yml + kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml + sleep 30s echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" # shellcheck disable=SC2199 @@ -75,6 +77,8 @@ for sync in ${SYNC_ARRAY[@]}; do kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml kubectl delete -f "${PRGDIR}"/shenyu-examples-http.yml kubectl delete -f "${PRGDIR}"/shenyu-rocketmq.yml + kubectl delete -f "${PRGDIR}"/shenyu-kafka.yml + # shellcheck disable=SC2199 # shellcheck disable=SC2076 if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml new file mode 100644 index 000000000000..15e2eb414a4b --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + namespace: default + labels: + app: zookeeper +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + image: wurstmeister/zookeeper:3.4.6 + ports: + - containerPort: 2181 + +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: default + labels: + app: zookeeper +spec: + ports: + - port: 2181 + name: client + selector: + app: zookeeper + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka + namespace: default + labels: + app: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + spec: + containers: + - name: kafka + image: wurstmeister/kafka:2.12-2.1.1 + env: + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://kafka:9092 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zookeeper:2181 + ports: + - containerPort: 9092 + +apiVersion: v1 +kind: Service +metadata: + name: kafka + namespace: default + labels: + app: kafka +spec: + ports: + - port: 9092 + name: client + protocol: TCP + targetPort: 9092 + nodePort: 31877 + + selector: + app: kafka diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml index 2a3141908e2b..7fa2a64563db 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml @@ -32,5 +32,10 @@ rocketmq-client 4.9.3 + + org.apache.kafka + kafka-clients + 2.4.1 + diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 256aca6f1109..c89c22ee9385 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -20,6 +20,10 @@ import com.google.common.collect.Lists; import io.restassured.http.Method; import org.apache.commons.collections.CollectionUtils; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; @@ -37,8 +41,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; import java.util.List; +import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; @@ -77,6 +86,67 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } + private ShenYuScenarioSpec testKafkaHello(){ + return ShenYuScenarioSpec.builder() + .name("testKafkaHello") + .beforeEachSpec( + ShenYuBeforeEachSpec.builder() + .addSelectorAndRule( + newSelectorBuilder("selector",Plugin.LOGGING_KAFKA) + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build(), + newRuleBuilder("rule") + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build() + ) + .checker(exists(TEST)) + .build() + ) + .caseSpec( + ShenYuCaseSpec.builder() + .add(request -> { + AtomicBoolean isLog = new AtomicBoolean(false); + try { + Thread.sleep(1000 * 30); + request.request(Method.GET, "/http/order/findById?id=23"); + Properties props = new Properties(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + "127.0.0.1:31877"); + KafkaConsumer consumer = new KafkaConsumer<>(props); + consumer.subscribe(Arrays.asList(TOPIC)); + AtomicReference keepCosuming = new AtomicReference<>(true); + Instant start = Instant.now(); + while(keepCosuming.get()){ + if (Duration.between(start, Instant.now()).toMillis() > 60000) { + keepCosuming.set(false); + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); + records.forEach(record ->{ + String message = record.value(); + if (message.contains("/http/order/findById?id=23")) { + isLog.set(true); + keepCosuming.set(false); + } + }); + } + Assertions.assertTrue(isLog.get()); + } catch (InterruptedException e) { + LOG.info("isLog.get():{}", isLog.get()); + LOG.error("error", e); + throw new RuntimeException(e); + } + }).build() + ).build(); + } + private ShenYuScenarioSpec testRocketMQHello() { return ShenYuScenarioSpec.builder() .name("testRocketMQHello") diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 309ff66bc58e..e92df5bf898f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -110,6 +110,16 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr formData.add("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("29", formData); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); + LOG.info("start loggingKafka plugin"); + formData.add("id","33"); + formData.add("name","loggingKafka"); + formData.add("enabled","true"); + formData.add("role","Logging"); + formData.add("sort","180"); + formData.add("config","{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + adminClient.changePluginStatus("33",formData); + WaitDataSync.waitGatewayPluginUse(gatewayClient,"org.apache.shenyu.plugin.logging.kafka"); + } @ShenYuScenario(provider = DividePluginCases.class) From aca9d87fc48854f274e9287c5902b7cb36fd2e99 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 31 Aug 2024 17:04:20 +0800 Subject: [PATCH 002/155] [type:feat]add kafka logging e2e test --- .../k8s/{shenu-kafka.yml => shenyu-kafka.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/{shenu-kafka.yml => shenyu-kafka.yml} (100%) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml similarity index 100% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenu-kafka.yml rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml From 8513fe1b5ac53e4f7379c9adc30e0668512d116f Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 10:58:39 +0800 Subject: [PATCH 003/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index bcb7b1b982b7..817f193233a9 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter + * @return the Shenyu cluster forward filter. */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From 1d21de716408702e85cf1ea02d91f03c9c09906e Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 12:44:54 +0800 Subject: [PATCH 004/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml index 15e2eb414a4b..e448ffb5ff88 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml @@ -87,6 +87,7 @@ metadata: labels: app: kafka spec: + type: NodePort ports: - port: 9092 name: client From c696c6c1cb6d594d9f0b2897b2a3f28cbe1603c3 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 15:02:28 +0800 Subject: [PATCH 005/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case-http/k8s/shenyu-kafka.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml index e448ffb5ff88..3e973beed141 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml @@ -37,6 +37,7 @@ spec: ports: - containerPort: 2181 +--- apiVersion: v1 kind: Service metadata: @@ -51,6 +52,7 @@ spec: selector: app: zookeeper +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -79,6 +81,7 @@ spec: ports: - containerPort: 9092 +--- apiVersion: v1 kind: Service metadata: @@ -94,6 +97,5 @@ spec: protocol: TCP targetPort: 9092 nodePort: 31877 - selector: - app: kafka + app: kafka \ No newline at end of file From 1f63829664aefbdc85b8728e3bbb8e7a382cf096 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 16:06:30 +0800 Subject: [PATCH 006/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index 817f193233a9..bcb7b1b982b7 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter. + * @return the Shenyu cluster forward filter */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From e8a2bd4c919d81ce8830e70f7095bdc3d85e0867 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 20:21:52 +0800 Subject: [PATCH 007/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml index 3e973beed141..93dd777c2353 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: zookeeper - image: wurstmeister/zookeeper:3.4.6 + image: zookeeper:3.7 ports: - containerPort: 2181 From 4a5252c7eca3053120da0f83c341e7a351132b3c Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Sep 2024 20:41:06 +0800 Subject: [PATCH 008/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index bcb7b1b982b7..817f193233a9 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter + * @return the Shenyu cluster forward filter. */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From 9b077b0e2e2ad10629c5d4bc1d7b3b4c6d15a45f Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 2 Sep 2024 21:31:24 +0800 Subject: [PATCH 009/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml index 93dd777c2353..0d95d1759997 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: kafka - image: wurstmeister/kafka:2.12-2.1.1 + image: bitnami/kafka:3.6.2 env: - name: KAFKA_ADVERTISED_LISTENERS value: PLAINTEXT://kafka:9092 From 507862046313de19bdaaf50c635f20ac9cd5610b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 2 Sep 2024 21:54:01 +0800 Subject: [PATCH 010/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index 817f193233a9..bcb7b1b982b7 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter. + * @return the Shenyu cluster forward filter */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From 56b53926b6532698bed7fe18615422e1b2efec81 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 2 Sep 2024 23:11:26 +0800 Subject: [PATCH 011/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/http/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/http/DividePluginTest.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index e94fd02c8c16..cbfa68c6680c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -84,7 +84,7 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } - private ShenYuScenarioSpec testKafkaHello(){ + private ShenYuScenarioSpec testKafkaHello() { return ShenYuScenarioSpec.builder() .name("testKafkaHello") .beforeEachSpec( diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index e92df5bf898f..4932854e2a33 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -111,14 +111,14 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr adminClient.changePluginStatus("29", formData); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); LOG.info("start loggingKafka plugin"); - formData.add("id","33"); - formData.add("name","loggingKafka"); - formData.add("enabled","true"); - formData.add("role","Logging"); - formData.add("sort","180"); + formData.add("id", "33"); + formData.add("name", "loggingKafka"); + formData.add("enabled", "true"); + formData.add("role", "Logging"); + formData.add("sort", "180"); formData.add("config","{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); - adminClient.changePluginStatus("33",formData); - WaitDataSync.waitGatewayPluginUse(gatewayClient,"org.apache.shenyu.plugin.logging.kafka"); + adminClient.changePluginStatus("33", formData); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } From 779cf79d18349d3f9c5e11075f25ba2def173b04 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 2 Sep 2024 23:26:44 +0800 Subject: [PATCH 012/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index bcb7b1b982b7..817f193233a9 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter + * @return the Shenyu cluster forward filter. */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From 93a8952111cd167dfa3ceecd089c25c37efbae67 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 8 Sep 2024 00:16:58 +0800 Subject: [PATCH 013/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/http/DividePluginCases.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index cbfa68c6680c..8f0efe9193a5 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -84,13 +84,13 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } - private ShenYuScenarioSpec testKafkaHello() { + private ShenYuScenarioSpec testKafkaHello() { return ShenYuScenarioSpec.builder() .name("testKafkaHello") .beforeEachSpec( ShenYuBeforeEachSpec.builder() .addSelectorAndRule( - newSelectorBuilder("selector",Plugin.LOGGING_KAFKA) + newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) .name("2") .matchMode(MatchMode.OR) .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) @@ -116,18 +116,18 @@ private ShenYuScenarioSpec testKafkaHello() { StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - "127.0.0.1:31877"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + ,"127.0.0.1:31877"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); - while(keepCosuming.get()){ + while (keepCosuming.get()) { if (Duration.between(start, Instant.now()).toMillis() > 60000) { keepCosuming.set(false); } ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); - records.forEach(record ->{ + records.forEach(record -> { String message = record.value(); if (message.contains("/http/order/findById?id=23")) { isLog.set(true); From 744ddf9576be7bfee491e243d37954fcf1a8c091 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 8 Sep 2024 20:23:51 +0800 Subject: [PATCH 014/155] [type:feat]add kafka logging e2e test --- .../main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java index 9bd5026202c7..88bc0ddcd0eb 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java @@ -30,7 +30,7 @@ public class ShenyuAdminBootstrap { /** * Main entrance. * - * @param args startup arguments + * @param args startup arguments. */ public static void main(final String[] args) { SpringApplication.run(ShenyuAdminBootstrap.class, args); From 0c265de800c6a1d8d8d1913002710f82d2ef3853 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Sep 2024 13:13:37 +0800 Subject: [PATCH 015/155] [type:feat]add kafka logging e2e test --- .../apache/shenyu/e2e/testcase/http/DividePluginCases.java | 6 +++--- .../apache/shenyu/e2e/testcase/http/DividePluginTest.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 8f0efe9193a5..518cb72b5236 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -68,7 +68,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello(), - testRocketMQHello() + testRocketMQHello(), + testKafkaHello() ); } @@ -116,8 +117,7 @@ private ShenYuScenarioSpec testKafkaHello() { StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG - ,"127.0.0.1:31877"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); AtomicReference keepCosuming = new AtomicReference<>(true); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 4932854e2a33..27ffe43a144c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -116,7 +116,8 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr formData.add("enabled", "true"); formData.add("role", "Logging"); formData.add("sort", "180"); - formData.add("config","{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + formData.add("config", + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("33", formData); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); From 3b005734d3ad9be84456ecb2f8800478c3b0264b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 17 Sep 2024 20:53:14 +0800 Subject: [PATCH 016/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/http/DividePluginTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 27ffe43a144c..97cf90a25a8f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -109,6 +109,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr formData.add("sort", "170"); formData.add("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("29", formData); + formData = new LinkedMultiValueMap<>(); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); LOG.info("start loggingKafka plugin"); formData.add("id", "33"); From 702073d62603fb9d4293c4d965fa88aa5ed3de69 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 14 Oct 2024 22:26:19 +0800 Subject: [PATCH 017/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/http/DividePluginTest.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 1b2ed522816f..42698217b0c8 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -113,14 +113,15 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); LOG.info("start loggingKafka plugin"); - formData.add("id", "33"); - formData.add("name", "loggingKafka"); - formData.add("enabled", "true"); - formData.add("role", "Logging"); - formData.add("sort", "180"); - formData.add("config", + reqBody.put("pluginId", "33"); + reqBody.put("name", "loggingKafka"); + reqBody.put("enabled", "true"); + reqBody.put("role", "Logging"); + reqBody.put("sort", "180"); + reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); - adminClient.changePluginStatus("33", formData); + adminClient.changePluginStatus("1801816010882822171", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } From f95f58c47196d0196613963f4de1bda328f36af0 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 27 Oct 2024 14:40:19 +0800 Subject: [PATCH 018/155] [type:feat]add kafka logging e2e test --- .../apache/shenyu/e2e/testcase/http/DividePluginCases.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 518cb72b5236..bd4a730f7567 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -68,8 +68,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello(), - testRocketMQHello(), - testKafkaHello() + testRocketMQHello() +// testKafkaHello() ); } From e99caa8c036bb3170f692f2e0fc396558fbddcf1 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 27 Oct 2024 14:40:19 +0800 Subject: [PATCH 019/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/http/DividePluginCases.java | 4 ++-- .../e2e/testcase/http/DividePluginTest.java | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 518cb72b5236..bd4a730f7567 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -68,8 +68,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello(), - testRocketMQHello(), - testKafkaHello() + testRocketMQHello() +// testKafkaHello() ); } diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index de1ceb45e675..40025e133916 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -115,17 +115,17 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); - LOG.info("start loggingKafka plugin"); - reqBody.put("pluginId", "33"); - reqBody.put("name", "loggingKafka"); - reqBody.put("enabled", "true"); - reqBody.put("role", "Logging"); - reqBody.put("sort", "180"); - reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); - reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); - adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); +// LOG.info("start loggingKafka plugin"); +// reqBody.put("pluginId", "33"); +// reqBody.put("name", "loggingKafka"); +// reqBody.put("enabled", "true"); +// reqBody.put("role", "Logging"); +// reqBody.put("sort", "180"); +// reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); +// reqBody.put("config", +// "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); +// adminClient.changePluginStatus("1801816010882822171", reqBody); +// WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } From 6fbdcde4360044b3e950dce06b9793183f0a92b3 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 27 Oct 2024 20:45:07 +0800 Subject: [PATCH 020/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-http-sync-compose.sh | 4 ++ .../compose/shenyu-kafka-compose.yml | 63 +++++++++++++++++++ .../k8s/script/e2e-http-sync.sh | 1 + 3 files changed, 68 insertions(+) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh index a0b102413c3d..c429798e7c7d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh @@ -38,6 +38,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health @@ -58,6 +59,9 @@ for sync in "${SYNC_ARRAY[@]}"; do echo "shenyu-rocketmq log:" echo "------------------" docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml logs + echo "shenyu-kafka log:" + echo "------------------" + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml logs echo "shenyu-examples-http log:" echo "------------------" docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml logs shenyu-examples-http diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml new file mode 100644 index 000000000000..922513db0fb2 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.9' + +services: + zookeeper: + image: zookeeper:latest + container_name: zookeeper + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=0.0.0.0:2888:3888 + ports: + - "31878:2181" + healthcheck: + test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] + interval: 10s + timeout: 2s + retries: 3 + start_period: 10s + restart: always + networks: + - shenyu + + kafka: + image: wurstmeister/kafka:latest + container_name: kafka + ports: + - "31877:9092" + healthcheck: + test: [ "CMD-SHELL", "wget -q -O - http://localhost:9092/actuator/health | grep UP || exit 1" ] + interval: 10s + timeout: 2s + retries: 3 + start_period: 10s + restart: always + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + depends_on: + - zookeeper + +networks: + shenyu: + name: shenyu + driver: bridge + external: true + diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh index d3e61c4f7e51..2c6549ae9755 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh @@ -38,6 +38,7 @@ for sync in ${SYNC_ARRAY[@]}; do kubectl apply -f "${PRGDIR}"/shenyu-rocketmq.yml kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health sleep 30s echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" From e51a5c58ea620e869a585dc2e1cd6f28346b8fab Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 30 Oct 2024 22:22:46 +0800 Subject: [PATCH 021/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/http/DividePluginCases.java | 4 +-- .../e2e/testcase/http/DividePluginTest.java | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index bd4a730f7567..518cb72b5236 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -68,8 +68,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello(), - testRocketMQHello() -// testKafkaHello() + testRocketMQHello(), + testKafkaHello() ); } diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 40025e133916..0868f564ea59 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -24,12 +24,14 @@ import org.apache.shenyu.e2e.constant.Constants; import org.apache.shenyu.e2e.engine.annotation.ShenYuScenario; import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; +import org.apache.shenyu.e2e.engine.scenario.specification.AfterEachSpec; import org.apache.shenyu.e2e.engine.scenario.specification.BeforeEachSpec; import org.apache.shenyu.e2e.engine.scenario.specification.CaseSpec; import org.apache.shenyu.e2e.enums.ServiceTypeEnum; import org.apache.shenyu.e2e.model.ResourcesData; import org.apache.shenyu.e2e.model.data.BindingData; import org.apache.shenyu.e2e.model.response.SelectorDTO; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; @@ -91,12 +93,12 @@ void before(final AdminClient client, final GatewayClient gateway, final BeforeE spec.getWaiting().waitFor(gateway); } -// @AfterEach -// void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { -// spec.getDeleter().delete(client, selectorIds); -// spec.deleteWaiting().waitFor(gateway); -// selectorIds = Lists.newArrayList(); -// } + @AfterEach + void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { + spec.getDeleter().delete(client, selectorIds); + spec.deleteWaiting().waitFor(gateway); + selectorIds = Lists.newArrayList(); + } @BeforeAll void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { @@ -115,17 +117,17 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); -// LOG.info("start loggingKafka plugin"); -// reqBody.put("pluginId", "33"); -// reqBody.put("name", "loggingKafka"); -// reqBody.put("enabled", "true"); -// reqBody.put("role", "Logging"); -// reqBody.put("sort", "180"); -// reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); -// reqBody.put("config", -// "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); -// adminClient.changePluginStatus("1801816010882822171", reqBody); -// WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + LOG.info("start loggingKafka plugin"); + reqBody.put("pluginId", "33"); + reqBody.put("name", "loggingKafka"); + reqBody.put("enabled", "true"); + reqBody.put("role", "Logging"); + reqBody.put("sort", "180"); + reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + reqBody.put("config", + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + adminClient.changePluginStatus("1801816010882822171", reqBody); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } From 9b70af6a6984da46090ee83738f5c500c4ab5414 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Thu, 7 Nov 2024 22:55:43 +0800 Subject: [PATCH 022/155] [type:feat]add kafka logging e2e test --- pom.xml | 1 + .../compose/script/e2e-http-sync-compose.sh | 4 - .../k8s/script/e2e-http-sync.sh | 2 - .../shenyu-e2e-case-http/pom.xml | 5 - .../e2e/testcase/http/DividePluginCases.java | 63 +------- .../e2e/testcase/http/DividePluginTest.java | 12 -- .../compose/script/e2e-http-sync-compose.sh | 66 +++++++++ .../compose/shenyu-kafka-compose.yml | 0 .../k8s/script/e2e-http-sync.sh | 81 ++++++++++ .../k8s/shenyu-kafka.yml | 0 .../shenyu-e2e-case-logging-kafka/pom.xml | 41 +++++ .../e2e/testcase/kafka/DataSynTest.java | 59 ++++++++ .../e2e/testcase/kafka/DividePluginCases.java | 140 ++++++++++++++++++ .../e2e/testcase/kafka/DividePluginTest.java | 128 ++++++++++++++++ 14 files changed, 517 insertions(+), 85 deletions(-) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh rename shenyu-e2e/shenyu-e2e-case/{shenyu-e2e-case-http => shenyu-e2e-case-logging-kafka}/compose/shenyu-kafka-compose.yml (100%) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh rename shenyu-e2e/shenyu-e2e-case/{shenyu-e2e-case-http => shenyu-e2e-case-logging-kafka}/k8s/shenyu-kafka.yml (100%) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java diff --git a/pom.xml b/pom.xml index c8603992517d..6c97afa637f8 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ shenyu-discovery shenyu-registry shenyu-kubernetes-controller + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh index c429798e7c7d..a0b102413c3d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh @@ -38,7 +38,6 @@ for sync in "${SYNC_ARRAY[@]}"; do sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull - docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health @@ -59,9 +58,6 @@ for sync in "${SYNC_ARRAY[@]}"; do echo "shenyu-rocketmq log:" echo "------------------" docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml logs - echo "shenyu-kafka log:" - echo "------------------" - docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml logs echo "shenyu-examples-http log:" echo "------------------" docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml logs shenyu-examples-http diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh index 2c6549ae9755..95d7931c0be5 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/script/e2e-http-sync.sh @@ -37,7 +37,6 @@ for sync in ${SYNC_ARRAY[@]}; do kubectl apply -f "${PRGDIR}"/shenyu-rocketmq.yml - kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health sleep 30s @@ -78,7 +77,6 @@ for sync in ${SYNC_ARRAY[@]}; do kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml kubectl delete -f "${PRGDIR}"/shenyu-examples-http.yml kubectl delete -f "${PRGDIR}"/shenyu-rocketmq.yml - kubectl delete -f "${PRGDIR}"/shenyu-kafka.yml # shellcheck disable=SC2199 # shellcheck disable=SC2076 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml index 7fa2a64563db..2a3141908e2b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml @@ -32,10 +32,5 @@ rocketmq-client 4.9.3 - - org.apache.kafka - kafka-clients - 2.4.1 - diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 518cb72b5236..09a43b366fb7 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -68,8 +68,7 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello(), - testRocketMQHello(), - testKafkaHello() + testRocketMQHello() ); } @@ -85,66 +84,6 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } - private ShenYuScenarioSpec testKafkaHello() { - return ShenYuScenarioSpec.builder() - .name("testKafkaHello") - .beforeEachSpec( - ShenYuBeforeEachSpec.builder() - .addSelectorAndRule( - newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) - .name("2") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build(), - newRuleBuilder("rule") - .name("2") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build() - ) - .checker(exists(TEST)) - .build() - ) - .caseSpec( - ShenYuCaseSpec.builder() - .add(request -> { - AtomicBoolean isLog = new AtomicBoolean(false); - try { - Thread.sleep(1000 * 30); - request.request(Method.GET, "/http/order/findById?id=23"); - Properties props = new Properties(); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); - KafkaConsumer consumer = new KafkaConsumer<>(props); - consumer.subscribe(Arrays.asList(TOPIC)); - AtomicReference keepCosuming = new AtomicReference<>(true); - Instant start = Instant.now(); - while (keepCosuming.get()) { - if (Duration.between(start, Instant.now()).toMillis() > 60000) { - keepCosuming.set(false); - } - ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); - records.forEach(record -> { - String message = record.value(); - if (message.contains("/http/order/findById?id=23")) { - isLog.set(true); - keepCosuming.set(false); - } - }); - } - Assertions.assertTrue(isLog.get()); - } catch (InterruptedException e) { - LOG.info("isLog.get():{}", isLog.get()); - LOG.error("error", e); - throw new RuntimeException(e); - } - }).build() - ).build(); - } - private ShenYuScenarioSpec testRocketMQHello() { return ShenYuScenarioSpec.builder() .name("testRocketMQHello") diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 0868f564ea59..f9deb5c7fc8a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -117,18 +117,6 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); - LOG.info("start loggingKafka plugin"); - reqBody.put("pluginId", "33"); - reqBody.put("name", "loggingKafka"); - reqBody.put("enabled", "true"); - reqBody.put("role", "Logging"); - reqBody.put("sort", "180"); - reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); - reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); - adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); - } @ShenYuScenario(provider = DividePluginCases.class) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh new file mode 100644 index 000000000000..4989f57a3c6b --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# init kubernetes for mysql +SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")") +bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh + +# init register center +CUR_PATH=$(readlink -f "$(dirname "$0")") +PRGDIR=$(dirname "$CUR_PATH") +# init shenyu sync +SYNC_ARRAY=("websocket" "http" "zookeeper" "etcd") +#SYNC_ARRAY=("websocket" "nacos") +#MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") + +docker network create -d bridge shenyu + +for sync in "${SYNC_ARRAY[@]}"; do + echo -e "------------------\n" + echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml up -d --quiet-pull + sleep 30s + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull + sleep 30s + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health + sleep 10s + docker ps -a + ## run e2e-test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-http -am test + # shellcheck disable=SC2181 + if (($?)); then + echo "${sync}-sync-e2e-test failed" + echo "------------------" + echo "shenyu-admin log:" + echo "------------------" + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml logs shenyu-admin + echo "shenyu-bootstrap log:" + echo "------------------" + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml logs shenyu-bootstrap + echo "shenyu-kafka log:" + echo "------------------" + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml logs + exit 1 + fi + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml down + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml down + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml down + echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " +done diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml similarity index 100% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/shenyu-kafka-compose.yml rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh new file mode 100644 index 000000000000..07650c592b8a --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +docker save shenyu-examples-http:latest | sudo k3s ctr images import - + +# init kubernetes for mysql +SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")") +bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh + +# init register center +CUR_PATH=$(readlink -f "$(dirname "$0")") +PRGDIR=$(dirname "$CUR_PATH") +kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-cm.yml + +# init shenyu sync +SYNC_ARRAY=("websocket" "http" "zookeeper" "etcd") +#SYNC_ARRAY=("websocket" "nacos") +MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") +for sync in ${SYNC_ARRAY[@]}; do + echo -e "------------------\n" + kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health + + sleep 30s + echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" + # shellcheck disable=SC2199 + # shellcheck disable=SC2076 + # shellcheck disable=SC2154 + if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml + sleep 10s + fi + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + sleep 10s + kubectl get pod -o wide + + kubectl logs "$(kubectl get pod -o wide | grep shenyu-admin | awk '{print $1}')" + + ## run e2e-test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-kafka -am test + # shellcheck disable=SC2181 + if (($?)); then + echo "${sync}-sync-e2e-test failed" + echo "shenyu-admin log:" + echo "------------------" + kubectl logs "$(kubectl get pod -o wide | grep shenyu-admin | awk '{print $1}')" + echo "shenyu-bootstrap log:" + echo "------------------" + kubectl logs "$(kubectl get pod -o wide | grep shenyu-bootstrap | awk '{print $1}')" + exit 1 + fi + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-mysql.yml + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml + kubectl delete -f "${PRGDIR}"/shenyu-kafka.yml + + # shellcheck disable=SC2199 + # shellcheck disable=SC2076 + if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml + fi + echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" +done diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml similarity index 100% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/k8s/shenyu-kafka.yml rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml new file mode 100644 index 000000000000..3404b82f5bed --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml @@ -0,0 +1,41 @@ + + + + + org.apache.shenyu + shenyu-e2e-case + 0.0.1-SNAPSHOT + + 4.0.0 + shenyu-e2e-case-logging-kafka + + + + org.apache.rocketmq + rocketmq-client + 4.9.3 + + + org.apache.kafka + kafka-clients + 2.4.1 + + + diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java new file mode 100644 index 000000000000..9b8262834139 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.kafka; + +import org.apache.shenyu.e2e.client.WaitDataSync; +import org.apache.shenyu.e2e.client.admin.AdminClient; +import org.apache.shenyu.e2e.client.gateway.GatewayClient; +import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; +import org.apache.shenyu.e2e.enums.ServiceTypeEnum; +import org.junit.jupiter.api.Test; + +/** + * Testing the correctness of etcd data synchronization method. + */ +@ShenYuTest(environments = { + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-admin", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31095", + type = ServiceTypeEnum.SHENYU_ADMIN, + parameters = { + @ShenYuTest.Parameter(key = "username", value = "admin"), + @ShenYuTest.Parameter(key = "password", value = "123456") + } + ) + ), + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-gateway", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31195", + type = ServiceTypeEnum.SHENYU_GATEWAY + ) + ) +}) +public class DataSynTest { + + @Test + void testDataSyn(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { + adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + } +} diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java new file mode 100644 index 000000000000..70fe2a0437bd --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.kafka; + +import com.google.common.collect.Lists; +import io.restassured.http.Method; +import org.apache.commons.collections.CollectionUtils; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.shenyu.e2e.engine.scenario.ShenYuScenarioProvider; +import org.apache.shenyu.e2e.engine.scenario.specification.ScenarioSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuBeforeEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuCaseSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuScenarioSpec; +import org.apache.shenyu.e2e.model.MatchMode; +import org.apache.shenyu.e2e.model.Plugin; +import org.apache.shenyu.e2e.model.data.Condition; +import org.junit.jupiter.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.*; + +public class DividePluginCases implements ShenYuScenarioProvider { + + private static final String TOPIC = "shenyu-access-logging"; + + private static final String TEST = "/http/order/findById?id=123"; + + private static final Logger LOG = LoggerFactory.getLogger(DividePluginCases.class); + + @Override + public List get() { + return Lists.newArrayList( + testDivideHello(), + testKafkaHello() + ); + } + + private ShenYuScenarioSpec testDivideHello() { + return ShenYuScenarioSpec.builder() + .name("http client hello1") + .beforeEachSpec(ShenYuBeforeEachSpec.builder() + .checker(exists("/http/order/findById?id=123")) + .build()) + .caseSpec(ShenYuCaseSpec.builder() + .addExists("/http/order/findById?id=123") + .build()) + .build(); + } + + private ShenYuScenarioSpec testKafkaHello() { + return ShenYuScenarioSpec.builder() + .name("testKafkaHello") + .beforeEachSpec( + ShenYuBeforeEachSpec.builder() + .addSelectorAndRule( + newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build(), + newRuleBuilder("rule") + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build() + ) + .checker(exists(TEST)) + .build() + ) + .caseSpec( + ShenYuCaseSpec.builder() + .add(request -> { + AtomicBoolean isLog = new AtomicBoolean(false); + try { + Thread.sleep(1000 * 30); + request.request(Method.GET, "/http/order/findById?id=23"); + Properties props = new Properties(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); + KafkaConsumer consumer = new KafkaConsumer<>(props); + consumer.subscribe(Arrays.asList(TOPIC)); + AtomicReference keepCosuming = new AtomicReference<>(true); + Instant start = Instant.now(); + while (keepCosuming.get()) { + if (Duration.between(start, Instant.now()).toMillis() > 60000) { + keepCosuming.set(false); + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); + records.forEach(record -> { + String message = record.value(); + if (message.contains("/http/order/findById?id=23")) { + isLog.set(true); + keepCosuming.set(false); + } + }); + } + Assertions.assertTrue(isLog.get()); + } catch (InterruptedException e) { + LOG.info("isLog.get():{}", isLog.get()); + LOG.error("error", e); + throw new RuntimeException(e); + } + }).build() + ).build(); + } +} diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java new file mode 100644 index 000000000000..69d4148335f7 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.kafka; + +import com.google.common.collect.Lists; +import org.apache.shenyu.e2e.client.WaitDataSync; +import org.apache.shenyu.e2e.client.admin.AdminClient; +import org.apache.shenyu.e2e.client.gateway.GatewayClient; +import org.apache.shenyu.e2e.constant.Constants; +import org.apache.shenyu.e2e.engine.annotation.ShenYuScenario; +import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; +import org.apache.shenyu.e2e.engine.scenario.specification.AfterEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.BeforeEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.CaseSpec; +import org.apache.shenyu.e2e.enums.ServiceTypeEnum; +import org.apache.shenyu.e2e.model.ResourcesData; +import org.apache.shenyu.e2e.model.data.BindingData; +import org.apache.shenyu.e2e.model.response.SelectorDTO; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.shenyu.e2e.constant.Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID; + +@ShenYuTest(environments = { + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-admin", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31095", + type = ServiceTypeEnum.SHENYU_ADMIN, + parameters = { + @ShenYuTest.Parameter(key = "username", value = "admin"), + @ShenYuTest.Parameter(key = "password", value = "123456") + } + ) + ), + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-gateway", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31195", + type = ServiceTypeEnum.SHENYU_GATEWAY + ) + ) +}) +public class DividePluginTest { + + private static final Logger LOG = LoggerFactory.getLogger(DividePluginTest.class); + + private List selectorIds = Lists.newArrayList(); + + @BeforeEach + void before(final AdminClient client, final GatewayClient gateway, final BeforeEachSpec spec) { + spec.getChecker().check(gateway); + + ResourcesData resources = spec.getResources(); + for (ResourcesData.Resource res : resources.getResources()) { + SelectorDTO dto = client.create(res.getSelector()); + selectorIds.add(dto.getId()); + res.getRules().forEach(rule -> { + rule.setSelectorId(dto.getId()); + client.create(rule); + }); + BindingData bindingData = res.getBindingData(); + if (Objects.nonNull(bindingData)) { + bindingData.setSelectorId(dto.getId()); + bindingData.setNamespaceId(SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + client.bindingData(bindingData); + } + } + + spec.getWaiting().waitFor(gateway); + } + + @AfterEach + void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { + spec.getDeleter().delete(client, selectorIds); + spec.deleteWaiting().waitFor(gateway); + selectorIds = Lists.newArrayList(); + } + + @BeforeAll + void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { + adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + Map reqBody = new HashMap<>(); + LOG.info("start loggingKafka plugin"); + reqBody.put("pluginId", "33"); + reqBody.put("name", "loggingKafka"); + reqBody.put("enabled", "true"); + reqBody.put("role", "Logging"); + reqBody.put("sort", "180"); + reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + reqBody.put("config", + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + adminClient.changePluginStatus("1801816010882822171", reqBody); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + + } + + @ShenYuScenario(provider = DividePluginCases.class) + void testDivide(final GatewayClient gateway, final CaseSpec spec) { + spec.getVerifiers().forEach(verifier -> verifier.verify(gateway.getHttpRequesterSupplier().get())); + } +} From 64bf1b95d85d0fec519ad8203fa97e0750550399 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 11 Nov 2024 12:27:41 +0800 Subject: [PATCH 023/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 89009f1ad00d..2afe2bf512ac 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -218,6 +218,8 @@ jobs: script: e2e-cluster-jdbc-compose - case: shenyu-e2e-case-cluster script: e2e-cluster-zookeeper-compose + - case: shenyu-e2e-case-kafka + script: shenyu-kafka-compose steps: - uses: actions/checkout@v2 From e77020b5bb06841250567bb95a448e0076de79e5 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 13 Nov 2024 09:32:14 +0800 Subject: [PATCH 024/155] [type:feat]add kafka logging e2e test --- shenyu-e2e/shenyu-e2e-case/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/pom.xml b/shenyu-e2e/shenyu-e2e-case/pom.xml index 901a5c593615..ecebd0d98da9 100644 --- a/shenyu-e2e/shenyu-e2e-case/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/pom.xml @@ -32,6 +32,7 @@ shenyu-e2e-case-cluster shenyu-e2e-case-storage shenyu-e2e-case-http + shenyu-e2e-case-logging-kafka shenyu-e2e-case-spring-cloud shenyu-e2e-case-apache-dubbo shenyu-e2e-case-sofa From ac0e34975ffd49fbb3666320958d399832022cda Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 16 Nov 2024 16:43:54 +0800 Subject: [PATCH 025/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/http/DividePluginCases.java | 10 +--------- .../shenyu-e2e-case-logging-kafka/pom.xml | 5 ----- .../shenyu/e2e/testcase/kafka/DividePluginCases.java | 8 +++----- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 09a43b366fb7..062f5d9c3025 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -20,10 +20,6 @@ import com.google.common.collect.Lists; import io.restassured.http.Method; import org.apache.commons.collections.CollectionUtils; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; @@ -39,13 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.Duration; -import java.time.Instant; -import java.util.Arrays; + import java.util.List; -import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml index 3404b82f5bed..72162ec5ed9d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml @@ -27,11 +27,6 @@ shenyu-e2e-case-logging-kafka - - org.apache.rocketmq - rocketmq-client - 4.9.3 - org.apache.kafka kafka-clients diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 70fe2a0437bd..89c6b7197a49 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -19,14 +19,10 @@ import com.google.common.collect.Lists; import io.restassured.http.Method; -import org.apache.commons.collections.CollectionUtils; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.shenyu.e2e.engine.scenario.ShenYuScenarioProvider; import org.apache.shenyu.e2e.engine.scenario.specification.ScenarioSpec; import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuBeforeEachSpec; @@ -48,7 +44,9 @@ import java.util.concurrent.atomic.AtomicReference; import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; -import static org.apache.shenyu.e2e.template.ResourceDataTemplate.*; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newRuleBuilder; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newSelectorBuilder; public class DividePluginCases implements ShenYuScenarioProvider { From b33a723f9f50eaa066866355406890a55d5821cf Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 16 Nov 2024 17:02:30 +0800 Subject: [PATCH 026/155] [type:feat]add kafka logging e2e test --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c97afa637f8..c8603992517d 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,6 @@ shenyu-discovery shenyu-registry shenyu-kubernetes-controller - shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka From 2e6f2739c843e7f0c3ce5b4a88069b066f15aa1b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 16 Nov 2024 18:50:50 +0800 Subject: [PATCH 027/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 2afe2bf512ac..2b50f8fa6dbe 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -218,7 +218,7 @@ jobs: script: e2e-cluster-jdbc-compose - case: shenyu-e2e-case-cluster script: e2e-cluster-zookeeper-compose - - case: shenyu-e2e-case-kafka + - case: shenyu-e2e-case-logging-kafka script: shenyu-kafka-compose steps: From b8427d0cd7148de5255e362ed9b2e26e97928657 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 17 Nov 2024 10:50:15 +0800 Subject: [PATCH 028/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 2 +- .../{e2e-http-sync-compose.sh => e2e-kafka-sync-compose.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/{e2e-http-sync-compose.sh => e2e-kafka-sync-compose.sh} (100%) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 2b50f8fa6dbe..44f99158f422 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -219,7 +219,7 @@ jobs: - case: shenyu-e2e-case-cluster script: e2e-cluster-zookeeper-compose - case: shenyu-e2e-case-logging-kafka - script: shenyu-kafka-compose + script: e2e-kafka-sync-compose steps: - uses: actions/checkout@v2 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh similarity index 100% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-http-sync-compose.sh rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh From 4d3c27d3df95ec0a876eec59ca81e15845ad3b1f Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 17 Nov 2024 13:51:38 +0800 Subject: [PATCH 029/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/http/DividePluginTest.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index f9deb5c7fc8a..1b18091e7c47 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -24,14 +24,12 @@ import org.apache.shenyu.e2e.constant.Constants; import org.apache.shenyu.e2e.engine.annotation.ShenYuScenario; import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; -import org.apache.shenyu.e2e.engine.scenario.specification.AfterEachSpec; import org.apache.shenyu.e2e.engine.scenario.specification.BeforeEachSpec; import org.apache.shenyu.e2e.engine.scenario.specification.CaseSpec; import org.apache.shenyu.e2e.enums.ServiceTypeEnum; import org.apache.shenyu.e2e.model.ResourcesData; import org.apache.shenyu.e2e.model.data.BindingData; import org.apache.shenyu.e2e.model.response.SelectorDTO; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; @@ -93,12 +91,12 @@ void before(final AdminClient client, final GatewayClient gateway, final BeforeE spec.getWaiting().waitFor(gateway); } - @AfterEach - void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { - spec.getDeleter().delete(client, selectorIds); - spec.deleteWaiting().waitFor(gateway); - selectorIds = Lists.newArrayList(); - } +// @AfterEach +// void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { +// spec.getDeleter().delete(client, selectorIds); +// spec.deleteWaiting().waitFor(gateway); +// selectorIds = Lists.newArrayList(); +// } @BeforeAll void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { From d57ed162d94a1cfceedc1d91dddf8a0b03f83997 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 17 Nov 2024 14:51:19 +0800 Subject: [PATCH 030/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index 4989f57a3c6b..6ac4d12f9d08 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -60,7 +60,6 @@ for sync in "${SYNC_ARRAY[@]}"; do exit 1 fi docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml down - docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml down docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml down echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " done From f6d1b535a9116674d767c80bb562e769b87d5183 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 20 Nov 2024 12:41:30 +0800 Subject: [PATCH 031/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 922513db0fb2..ae77048ddcd4 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -18,7 +18,7 @@ version: '3.9' services: zookeeper: - image: zookeeper:latest + image: zookeeper:3.8.4 container_name: zookeeper environment: ZOO_MY_ID: 1 @@ -31,6 +31,8 @@ services: timeout: 2s retries: 3 start_period: 10s + audit: + restart: always networks: - shenyu From 152563d4a5c984ab4c2e2c0bdfb3451b61312b45 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 20 Nov 2024 23:33:22 +0800 Subject: [PATCH 032/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 ++-- .../shenyu-e2e-case-logging-kafka/compose/zoo.cfg | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index ae77048ddcd4..f9ef0840302b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -25,14 +25,14 @@ services: ZOO_SERVERS: server.1=0.0.0.0:2888:3888 ports: - "31878:2181" + volumes: + - ./zoo.cfg healthcheck: test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] interval: 10s timeout: 2s retries: 3 start_period: 10s - audit: - restart: always networks: - shenyu diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg new file mode 100644 index 000000000000..396bc529c974 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg @@ -0,0 +1,3 @@ +tickTime=2000 +dataDir=/var/lib/zookeeper/data +audit.enable=true \ No newline at end of file From 61e276b578f5f778e91d05e7a183a2d255a7846e Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 15:13:04 +0800 Subject: [PATCH 033/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index f9ef0840302b..ea1b1098894d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -26,7 +26,7 @@ services: ports: - "31878:2181" volumes: - - ./zoo.cfg + - /zoo.cfg:/conf/zoo.cfg healthcheck: test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] interval: 10s From bf22e4a8102f2d9328e7098bea4448de3e9a12f5 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 15:21:44 +0800 Subject: [PATCH 034/155] [type:feat]add kafka logging e2e test --- .../compose/zoo.cfg | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg index 396bc529c974..3ab1a8615efc 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + tickTime=2000 dataDir=/var/lib/zookeeper/data audit.enable=true \ No newline at end of file From cf4b474407694af790bf4f103a74a907a20d3f6b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 19:15:18 +0800 Subject: [PATCH 035/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index ea1b1098894d..d3afa7c6e994 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -26,7 +26,7 @@ services: ports: - "31878:2181" volumes: - - /zoo.cfg:/conf/zoo.cfg + - /zoo.cfg:/conf/zookeeper.cfg healthcheck: test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] interval: 10s From 6a05800d470fe28d3e5a935c0b590b5f8f45218d Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 21:10:51 +0800 Subject: [PATCH 036/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index d3afa7c6e994..4d8218a9389b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -14,15 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: '3.9' services: zookeeper: image: zookeeper:3.8.4 container_name: zookeeper environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=0.0.0.0:2888:3888 + ZOO_STANDALONE_ENABLED: "true" ports: - "31878:2181" volumes: From cba50970130297f1753e03bf268a74756c274a00 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 22:04:37 +0800 Subject: [PATCH 037/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 4d8218a9389b..9b0fe4eeab4f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -46,6 +46,8 @@ services: timeout: 2s retries: 3 start_period: 10s + depends_on: + - zookeeper restart: always environment: KAFKA_BROKER_ID: 1 From e2c21800825795515abc2b299a79cfd684f4e957 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 22:41:30 +0800 Subject: [PATCH 038/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 9b0fe4eeab4f..892231df89fb 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -40,14 +40,15 @@ services: container_name: kafka ports: - "31877:9092" - healthcheck: - test: [ "CMD-SHELL", "wget -q -O - http://localhost:9092/actuator/health | grep UP || exit 1" ] - interval: 10s - timeout: 2s - retries: 3 - start_period: 10s - depends_on: - - zookeeper +# healthcheck: +# test: [ "CMD-SHELL", "wget -q -O - http://localhost:9092/actuator/health | grep UP || exit 1" ] +# interval: 10s +# timeout: 2s +# retries: 3 +# start_period: 10s +# depends_on: +# zookeeper: +# condition: service_healthy restart: always environment: KAFKA_BROKER_ID: 1 From 9bfad2b5ebedcd99750cd4aa8f70664a10d840b7 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 23:05:26 +0800 Subject: [PATCH 039/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 892231df89fb..7c517d911e3a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -25,12 +25,12 @@ services: - "31878:2181" volumes: - /zoo.cfg:/conf/zookeeper.cfg - healthcheck: - test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] - interval: 10s - timeout: 2s - retries: 3 - start_period: 10s +# healthcheck: +# test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] +# interval: 10s +# timeout: 2s +# retries: 3 +# start_period: 10s restart: always networks: - shenyu From f4b9b993f9ac00b2c46e1739417496ee10a0b2ef Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 23 Nov 2024 23:33:57 +0800 Subject: [PATCH 040/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 7c517d911e3a..a0fbe7dcf719 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -23,8 +23,8 @@ services: ZOO_STANDALONE_ENABLED: "true" ports: - "31878:2181" - volumes: - - /zoo.cfg:/conf/zookeeper.cfg +# volumes: +# - /zoo.cfg:/conf/zookeeper.cfg # healthcheck: # test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] # interval: 10s @@ -57,6 +57,8 @@ services: KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 depends_on: - zookeeper + networks: + - shenyu networks: shenyu: From 60b7e482e8e582a46674a7478df4154738892244 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 24 Nov 2024 13:44:29 +0800 Subject: [PATCH 041/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/admin/config/ClusterConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java index 817f193233a9..bcb7b1b982b7 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/ClusterConfiguration.java @@ -72,7 +72,7 @@ public ShenyuRunningModeService shenyuRunningModeService(final ClusterSelectMast * Shenyu cluster forward filter. * * @param clusterProperties cluster properties - * @return the Shenyu cluster forward filter. + * @return the Shenyu cluster forward filter */ @Bean public ClusterForwardFilter clusterForwardFilter(final ClusterProperties clusterProperties) { From 5a8deba141c61a6bfd89ff320b579674235d690a Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 24 Nov 2024 20:49:05 +0800 Subject: [PATCH 042/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 89c6b7197a49..6216bdd3375c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -34,7 +34,6 @@ import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -42,7 +41,6 @@ import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; - import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newRuleBuilder; From 3be3a63037b8d62b8793f2ad578974320772c835 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 24 Nov 2024 22:36:53 +0800 Subject: [PATCH 043/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index 6ac4d12f9d08..a08461275f8a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -37,6 +37,8 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull + sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health @@ -61,5 +63,6 @@ for sync in "${SYNC_ARRAY[@]}"; do fi docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml down docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml down + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml down echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " done From 4b1e94f58a116575b9647be00b19e99f772edbb3 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 25 Nov 2024 11:02:19 +0800 Subject: [PATCH 044/155] [type:fix] fix zookeeper healthy check --- .../compose/shenyu-kafka-compose.yml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index a0fbe7dcf719..5472802a3e74 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -17,27 +17,28 @@ services: zookeeper: - image: zookeeper:3.8.4 - container_name: zookeeper + image: bitnami/zookeeper:latest + container_name: shenyu-zookeeper environment: ZOO_STANDALONE_ENABLED: "true" + ALLOW_ANONYMOUS_LOGIN: "yes" ports: - "31878:2181" # volumes: # - /zoo.cfg:/conf/zookeeper.cfg -# healthcheck: -# test: [ "CMD-SHELL", "wget -q -O - http://localhost:2181/actuator/health | grep UP || exit 1" ] -# interval: 10s -# timeout: 2s -# retries: 3 -# start_period: 10s + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 2181 || exit 1" ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 40s restart: always networks: - shenyu kafka: image: wurstmeister/kafka:latest - container_name: kafka + container_name: shenyu-kafka ports: - "31877:9092" # healthcheck: @@ -56,7 +57,8 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 depends_on: - - zookeeper + zookeeper: + condition: service_healthy networks: - shenyu From 4c7ae0644edcafd13e8e8158ab4d154c618d2dd7 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 25 Nov 2024 22:19:31 +0800 Subject: [PATCH 045/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index a08461275f8a..eb6bc0135bea 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -37,7 +37,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull + docker compose -f "$SHENYU_TESTCASE_DIR"/shenyu-e2e-case-http/compose/script/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull sleep 30s From e15a502408766c9fd38b0119582dac51da998898 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 25 Nov 2024 23:03:57 +0800 Subject: [PATCH 046/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 2 +- .../compose/shenyu-examples-http-compose.yml | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-examples-http-compose.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index eb6bc0135bea..a08461275f8a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -37,7 +37,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "$SHENYU_TESTCASE_DIR"/shenyu-e2e-case-http/compose/script/shenyu-examples-http-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull sleep 30s diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-examples-http-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-examples-http-compose.yml new file mode 100644 index 000000000000..ce8c8a7e7b22 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-examples-http-compose.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.9' + +services: + shenyu-examples-http: + image: shenyu-examples-http:latest + container_name: shenyu-examples-http + environment: + - shenyu.register.serverLists=http://shenyu-admin:9095 + ports: + - "31189:8189" + healthcheck: + test: [ "CMD-SHELL", "wget -q -O - http://localhost:8189/actuator/health | grep UP || exit 1" ] + interval: 10s + timeout: 2s + retries: 3 + start_period: 10s + restart: always + networks: + - shenyu + +networks: + shenyu: + name: shenyu + driver: bridge + external: true From da41d29a7109988ec2907a17580b34f4ba956519 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 25 Nov 2024 23:43:45 +0800 Subject: [PATCH 047/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/kafka/DividePluginCases.java | 118 +++++++++--------- .../e2e/testcase/kafka/DividePluginTest.java | 1 - 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 6216bdd3375c..09d5d4533ec1 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -74,63 +74,63 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } - private ShenYuScenarioSpec testKafkaHello() { - return ShenYuScenarioSpec.builder() - .name("testKafkaHello") - .beforeEachSpec( - ShenYuBeforeEachSpec.builder() - .addSelectorAndRule( - newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) - .name("2") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build(), - newRuleBuilder("rule") - .name("2") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build() - ) - .checker(exists(TEST)) - .build() - ) - .caseSpec( - ShenYuCaseSpec.builder() - .add(request -> { - AtomicBoolean isLog = new AtomicBoolean(false); - try { - Thread.sleep(1000 * 30); - request.request(Method.GET, "/http/order/findById?id=23"); - Properties props = new Properties(); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); - KafkaConsumer consumer = new KafkaConsumer<>(props); - consumer.subscribe(Arrays.asList(TOPIC)); - AtomicReference keepCosuming = new AtomicReference<>(true); - Instant start = Instant.now(); - while (keepCosuming.get()) { - if (Duration.between(start, Instant.now()).toMillis() > 60000) { - keepCosuming.set(false); - } - ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); - records.forEach(record -> { - String message = record.value(); - if (message.contains("/http/order/findById?id=23")) { - isLog.set(true); - keepCosuming.set(false); - } - }); - } - Assertions.assertTrue(isLog.get()); - } catch (InterruptedException e) { - LOG.info("isLog.get():{}", isLog.get()); - LOG.error("error", e); - throw new RuntimeException(e); - } - }).build() - ).build(); - } +// private ShenYuScenarioSpec testKafkaHello() { +// return ShenYuScenarioSpec.builder() +// .name("testKafkaHello") +// .beforeEachSpec( +// ShenYuBeforeEachSpec.builder() +// .addSelectorAndRule( +// newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) +// .name("2") +// .matchMode(MatchMode.OR) +// .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) +// .build(), +// newRuleBuilder("rule") +// .name("2") +// .matchMode(MatchMode.OR) +// .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) +// .build() +// ) +// .checker(exists(TEST)) +// .build() +// ) +// .caseSpec( +// ShenYuCaseSpec.builder() +// .add(request -> { +// AtomicBoolean isLog = new AtomicBoolean(false); +// try { +// Thread.sleep(1000 * 30); +// request.request(Method.GET, "/http/order/findById?id=23"); +// Properties props = new Properties(); +// props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, +// StringDeserializer.class.getName()); +// props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, +// StringDeserializer.class.getName()); +// props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); +// KafkaConsumer consumer = new KafkaConsumer<>(props); +// consumer.subscribe(Arrays.asList(TOPIC)); +// AtomicReference keepCosuming = new AtomicReference<>(true); +// Instant start = Instant.now(); +// while (keepCosuming.get()) { +// if (Duration.between(start, Instant.now()).toMillis() > 60000) { +// keepCosuming.set(false); +// } +// ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); +// records.forEach(record -> { +// String message = record.value(); +// if (message.contains("/http/order/findById?id=23")) { +// isLog.set(true); +// keepCosuming.set(false); +// } +// }); +// } +// Assertions.assertTrue(isLog.get()); +// } catch (InterruptedException e) { +// LOG.info("isLog.get():{}", isLog.get()); +// LOG.error("error", e); +// throw new RuntimeException(e); +// } +// }).build() +// ).build(); +// } } diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index 69d4148335f7..ca9521c134df 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -118,7 +118,6 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); - } @ShenYuScenario(provider = DividePluginCases.class) From 02c5bd9dccd119cc8f468291cae39d9a97a5af21 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 26 Nov 2024 08:52:11 +0800 Subject: [PATCH 048/155] [type:fix] fix http rocketmq --- .../compose/script/e2e-kafka-sync-compose.sh | 1 + .../compose/shenyu-rocketmq-compose.yml | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index a08461275f8a..dac768da6546 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -37,6 +37,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml new file mode 100644 index 000000000000..c838d21a222e --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.9' + +services: + rocketmq-dialevoneid: + image: rocketmqinc/rocketmq:4.4.0 + container_name: rocketmq-dialevoneid + command: [ "/bin/sh", "mqnamesrv" ] + ports: + - "31876:9876" + environment: + - TZ=Asia/Shanghai + restart: always + networks: + - shenyu + + rocketmq-broker: + image: rocketmqinc/rocketmq:4.4.0 + container_name: rocketmq-broker + command: [ "/bin/sh", "mqbroker" ] + ports: + - "10909:10909" + - "10911:10911" + - "10912:10912" + environment: + - NAMESRV_ADDR=rocketmq-dialevoneid:9876 + - TZ=Asia/Shanghai + restart: always + networks: + - shenyu + +networks: + shenyu: + name: shenyu + driver: bridge + external: true \ No newline at end of file From a3d541be873581986e1b1e31260eb972cfaaac32 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 26 Nov 2024 22:19:51 +0800 Subject: [PATCH 049/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/kafka/DividePluginCases.java | 118 +++++++++--------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 09d5d4533ec1..6216bdd3375c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -74,63 +74,63 @@ private ShenYuScenarioSpec testDivideHello() { .build(); } -// private ShenYuScenarioSpec testKafkaHello() { -// return ShenYuScenarioSpec.builder() -// .name("testKafkaHello") -// .beforeEachSpec( -// ShenYuBeforeEachSpec.builder() -// .addSelectorAndRule( -// newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) -// .name("2") -// .matchMode(MatchMode.OR) -// .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) -// .build(), -// newRuleBuilder("rule") -// .name("2") -// .matchMode(MatchMode.OR) -// .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) -// .build() -// ) -// .checker(exists(TEST)) -// .build() -// ) -// .caseSpec( -// ShenYuCaseSpec.builder() -// .add(request -> { -// AtomicBoolean isLog = new AtomicBoolean(false); -// try { -// Thread.sleep(1000 * 30); -// request.request(Method.GET, "/http/order/findById?id=23"); -// Properties props = new Properties(); -// props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, -// StringDeserializer.class.getName()); -// props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, -// StringDeserializer.class.getName()); -// props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); -// KafkaConsumer consumer = new KafkaConsumer<>(props); -// consumer.subscribe(Arrays.asList(TOPIC)); -// AtomicReference keepCosuming = new AtomicReference<>(true); -// Instant start = Instant.now(); -// while (keepCosuming.get()) { -// if (Duration.between(start, Instant.now()).toMillis() > 60000) { -// keepCosuming.set(false); -// } -// ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); -// records.forEach(record -> { -// String message = record.value(); -// if (message.contains("/http/order/findById?id=23")) { -// isLog.set(true); -// keepCosuming.set(false); -// } -// }); -// } -// Assertions.assertTrue(isLog.get()); -// } catch (InterruptedException e) { -// LOG.info("isLog.get():{}", isLog.get()); -// LOG.error("error", e); -// throw new RuntimeException(e); -// } -// }).build() -// ).build(); -// } + private ShenYuScenarioSpec testKafkaHello() { + return ShenYuScenarioSpec.builder() + .name("testKafkaHello") + .beforeEachSpec( + ShenYuBeforeEachSpec.builder() + .addSelectorAndRule( + newSelectorBuilder("selector", Plugin.LOGGING_KAFKA) + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build(), + newRuleBuilder("rule") + .name("2") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build() + ) + .checker(exists(TEST)) + .build() + ) + .caseSpec( + ShenYuCaseSpec.builder() + .add(request -> { + AtomicBoolean isLog = new AtomicBoolean(false); + try { + Thread.sleep(1000 * 30); + request.request(Method.GET, "/http/order/findById?id=23"); + Properties props = new Properties(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); + KafkaConsumer consumer = new KafkaConsumer<>(props); + consumer.subscribe(Arrays.asList(TOPIC)); + AtomicReference keepCosuming = new AtomicReference<>(true); + Instant start = Instant.now(); + while (keepCosuming.get()) { + if (Duration.between(start, Instant.now()).toMillis() > 60000) { + keepCosuming.set(false); + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); + records.forEach(record -> { + String message = record.value(); + if (message.contains("/http/order/findById?id=23")) { + isLog.set(true); + keepCosuming.set(false); + } + }); + } + Assertions.assertTrue(isLog.get()); + } catch (InterruptedException e) { + LOG.info("isLog.get():{}", isLog.get()); + LOG.error("error", e); + throw new RuntimeException(e); + } + }).build() + ).build(); + } } From 7452eb6b6f4d3fd09c493daf4d3c8ef54f6749a2 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 26 Nov 2024 22:48:28 +0800 Subject: [PATCH 050/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 -- .../compose/zoo.cfg | 20 ------------------- 2 files changed, 22 deletions(-) delete mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 5472802a3e74..458aa8b8cf7e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -24,8 +24,6 @@ services: ALLOW_ANONYMOUS_LOGIN: "yes" ports: - "31878:2181" -# volumes: -# - /zoo.cfg:/conf/zookeeper.cfg healthcheck: test: [ "CMD-SHELL", "nc -z localhost 2181 || exit 1" ] interval: 30s diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg deleted file mode 100644 index 3ab1a8615efc..000000000000 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/zoo.cfg +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -tickTime=2000 -dataDir=/var/lib/zookeeper/data -audit.enable=true \ No newline at end of file From 2685fa0afbb289245ddcf5ed9dff8eeea5a0771e Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 26 Nov 2024 23:29:38 +0800 Subject: [PATCH 051/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 2 +- .../compose/shenyu-rocketmq-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index dac768da6546..f810a9015bf0 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -37,7 +37,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull +# docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml index c838d21a222e..5f34a739ca68 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml @@ -28,7 +28,7 @@ services: restart: always networks: - shenyu - + rocketmq-broker: image: rocketmqinc/rocketmq:4.4.0 container_name: rocketmq-broker From e080ed73b46542df33f700f209c198f4c515d962 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Fri, 29 Nov 2024 09:55:26 +0800 Subject: [PATCH 052/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-kafka-sync-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh index f810a9015bf0..725064463c7d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh @@ -46,7 +46,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 10s docker ps -a ## run e2e-test - ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-http -am test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-kafka -am test # shellcheck disable=SC2181 if (($?)); then echo "${sync}-sync-e2e-test failed" From 26d3fa66f12f84bf15edd1100d2d05b45c74cb9a Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 30 Nov 2024 09:52:23 +0800 Subject: [PATCH 053/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 6216bdd3375c..609b71960d89 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -106,6 +106,7 @@ private ShenYuScenarioSpec testKafkaHello() { StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); From a445faacad954cd72a1fb0bb61d324dc8e6948a8 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 30 Nov 2024 13:32:39 +0800 Subject: [PATCH 054/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java index 609b71960d89..4d1adcd2a20e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:31877"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:31877"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); AtomicReference keepCosuming = new AtomicReference<>(true); From 093126a1e686424ed60aa65a2b75e39cd0e2004d Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Dec 2024 09:18:36 +0800 Subject: [PATCH 055/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index ca9521c134df..cc750706c0ba 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -115,7 +115,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } From a56b11f1c31b9a2521f16f8ee830c04b592fd5f9 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Dec 2024 10:22:43 +0800 Subject: [PATCH 056/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/http/DividePluginTest.java | 1 + .../org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 1b18091e7c47..e92fdac27cf6 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -32,6 +32,7 @@ import org.apache.shenyu.e2e.model.response.SelectorDTO; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index cc750706c0ba..49756a90bf1c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.loggingKafka"); } @ShenYuScenario(provider = DividePluginCases.class) From 840cf939c9e063307d1625dfc2203b979f361b17 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Dec 2024 11:17:49 +0800 Subject: [PATCH 057/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/http/DividePluginTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index e92fdac27cf6..521a0df16a28 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -99,6 +99,7 @@ void before(final AdminClient client, final GatewayClient gateway, final BeforeE // selectorIds = Lists.newArrayList(); // } + @BeforeAll void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { adminClient.login(); From 7bc358c1ae6f7cfb71325cd718906a8d4b7ffc2b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Dec 2024 12:17:26 +0800 Subject: [PATCH 058/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/http/DividePluginTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 521a0df16a28..6bbe62f37e9f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -32,7 +32,6 @@ import org.apache.shenyu.e2e.model.response.SelectorDTO; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From a708a37eee52e15ab0b4de99807efc84556b0ff9 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 1 Dec 2024 20:36:53 +0800 Subject: [PATCH 059/155] [type:feat]add kafka logging e2e test --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index 49756a90bf1c..cc750706c0ba 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.loggingKafka"); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } @ShenYuScenario(provider = DividePluginCases.class) From bb3d35fe9232b0e26851aafc67800013b367f9a3 Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 10:33:18 +0800 Subject: [PATCH 060/155] kafka e2e --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index cc750706c0ba..73172d72fb97 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka.LoggingKafkaPlugin"); } @ShenYuScenario(provider = DividePluginCases.class) From 964e4d3222296dd3e0b7a8f7cb92bb3e8025c743 Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 10:55:38 +0800 Subject: [PATCH 061/155] kafka e2e --- .../org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index 73172d72fb97..cc750706c0ba 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka.LoggingKafkaPlugin"); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); } @ShenYuScenario(provider = DividePluginCases.class) From fe82363f4ab818cac070112af9e84d7322a76ad1 Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 10:57:14 +0800 Subject: [PATCH 062/155] kafka e2e --- .../main/java/org/apache/shenyu/e2e/client/WaitDataSync.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java index ce3efdd91c9a..172b7b987a36 100644 --- a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java +++ b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java @@ -17,6 +17,7 @@ package org.apache.shenyu.e2e.client; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.shenyu.e2e.client.admin.AdminClient; import org.apache.shenyu.e2e.client.gateway.GatewayClient; import org.junit.jupiter.api.Assertions; @@ -34,6 +35,8 @@ public class WaitDataSync { private static final Logger LOGGER = LoggerFactory.getLogger(WaitDataSync.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); /** * waitAdmin2GatewayDataSync. @@ -81,6 +84,7 @@ public static , U extends List> void waitAdmin2GatewayDataS */ public static void waitGatewayPluginUse(final GatewayClient gatewayClient, final String pluginClass) throws Exception { Map pluginMap = gatewayClient.getPlugins(); + LOGGER.info("pluginMap:{}", MAPPER.writeValueAsString(pluginMap)); int retryNum = 0; boolean existPlugin = false; while (!existPlugin && retryNum < 5) { @@ -93,6 +97,7 @@ public static void waitGatewayPluginUse(final GatewayClient gatewayClient, final Thread.sleep(10000); retryNum++; pluginMap = gatewayClient.getPlugins(); + LOGGER.info("pluginMap:{}", MAPPER.writeValueAsString(pluginMap)); } if (!existPlugin) { throw new AssertionFailedError(pluginClass + " plugin not found"); From 07838dd3864cda64d88c8d4817901b97ba9d2cbb Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 11:15:11 +0800 Subject: [PATCH 063/155] kafka e2e --- .../compose/shenyu-kafka-compose.yml | 8 ++++---- .../shenyu/e2e/testcase/kafka/DividePluginTest.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 458aa8b8cf7e..05fe00fe7ca9 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -35,7 +35,7 @@ services: - shenyu kafka: - image: wurstmeister/kafka:latest + image: bitnami/kafka:latest container_name: shenyu-kafka ports: - "31877:9092" @@ -51,9 +51,9 @@ services: restart: always environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:31877 + KAFKA_LISTENERS: PLAINTEXT://shenyu-kafka:31877 depends_on: zookeeper: condition: service_healthy diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index cc750706c0ba..cb7e9a84c6de 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31095", + baseUrl = "http://localhost:9095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31195", + baseUrl = "http://localhost:9195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) From 8214160940a138c498a19e87ddd0b3483819052e Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 11:34:08 +0800 Subject: [PATCH 064/155] spilt logging rocketmq and logging kafka --- .github/workflows/e2e-k8s.yml | 4 +- shenyu-e2e/shenyu-e2e-case/pom.xml | 1 + .../compose/script/e2e-http-sync-compose.sh | 4 - .../shenyu-e2e-case-http/pom.xml | 8 -- .../e2e/testcase/http/DividePluginCases.java | 87 +---------- .../e2e/testcase/http/DividePluginTest.java | 11 -- ...ompose.sh => e2e-logging-kafka-compose.sh} | 1 - .../script/e2e-logging-rocketmq-compose.sh | 67 +++++++++ .../compose/shenyu-examples-http-compose.yml | 41 ++++++ .../compose/shenyu-rocketmq-compose.yml | 0 .../k8s/script/e2e-http-sync.sh | 81 +++++++++++ .../k8s/shenyu-kafka.yml | 101 +++++++++++++ .../shenyu-e2e-case-logging-rocketmq/pom.xml | 36 +++++ .../logging/rocketmq/DataSynTest.java | 59 ++++++++ .../logging/rocketmq/DividePluginCases.java | 135 ++++++++++++++++++ .../logging/rocketmq/DividePluginTest.java | 126 ++++++++++++++++ 16 files changed, 651 insertions(+), 111 deletions(-) rename shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/{e2e-kafka-sync-compose.sh => e2e-logging-kafka-compose.sh} (97%) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-examples-http-compose.yml rename shenyu-e2e/shenyu-e2e-case/{shenyu-e2e-case-logging-kafka => shenyu-e2e-case-logging-rocketmq}/compose/shenyu-rocketmq-compose.yml (100%) create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/pom.xml create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginCases.java create mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 44f99158f422..bb57c2dfed10 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -218,8 +218,10 @@ jobs: script: e2e-cluster-jdbc-compose - case: shenyu-e2e-case-cluster script: e2e-cluster-zookeeper-compose + - case: shenyu-e2e-case-logging-rocketmq + script: e2e-logging-rocketmq-compose - case: shenyu-e2e-case-logging-kafka - script: e2e-kafka-sync-compose + script: e2e-logging-kafka-compose steps: - uses: actions/checkout@v2 diff --git a/shenyu-e2e/shenyu-e2e-case/pom.xml b/shenyu-e2e/shenyu-e2e-case/pom.xml index ecebd0d98da9..4ca3cc3d3127 100644 --- a/shenyu-e2e/shenyu-e2e-case/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/pom.xml @@ -33,6 +33,7 @@ shenyu-e2e-case-storage shenyu-e2e-case-http shenyu-e2e-case-logging-kafka + shenyu-e2e-case-logging-rocketmq shenyu-e2e-case-spring-cloud shenyu-e2e-case-apache-dubbo shenyu-e2e-case-sofa diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh index a0b102413c3d..446dae7dfb93 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/compose/script/e2e-http-sync-compose.sh @@ -37,7 +37,6 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health @@ -55,9 +54,6 @@ for sync in "${SYNC_ARRAY[@]}"; do echo "shenyu-bootstrap log:" echo "------------------" docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml logs shenyu-bootstrap - echo "shenyu-rocketmq log:" - echo "------------------" - docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml logs echo "shenyu-examples-http log:" echo "------------------" docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml logs shenyu-examples-http diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml index 2a3141908e2b..f326c8600958 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/pom.xml @@ -25,12 +25,4 @@ 4.0.0 shenyu-e2e-case-http - - - - org.apache.rocketmq - rocketmq-client - 4.9.3 - - diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java index 062f5d9c3025..b6e3f2aab051 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginCases.java @@ -18,49 +18,22 @@ package org.apache.shenyu.e2e.testcase.http; import com.google.common.collect.Lists; -import io.restassured.http.Method; -import org.apache.commons.collections.CollectionUtils; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.shenyu.e2e.engine.scenario.ShenYuScenarioProvider; import org.apache.shenyu.e2e.engine.scenario.specification.ScenarioSpec; import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuBeforeEachSpec; import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuCaseSpec; import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuScenarioSpec; -import org.apache.shenyu.e2e.model.MatchMode; -import org.apache.shenyu.e2e.model.Plugin; -import org.apache.shenyu.e2e.model.data.Condition; -import org.junit.jupiter.api.Assertions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; -import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; -import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newRuleBuilder; -import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newSelectorBuilder; public class DividePluginCases implements ShenYuScenarioProvider { - private static final String NAMESERVER = "http://localhost:31876"; - - private static final String CONSUMERGROUP = "shenyu-plugin-logging-rocketmq"; - - private static final String TOPIC = "shenyu-access-logging"; - - private static final String TEST = "/http/order/findById?id=123"; - - private static final Logger LOG = LoggerFactory.getLogger(DividePluginCases.class); - @Override public List get() { return Lists.newArrayList( - testDivideHello(), - testRocketMQHello() + testDivideHello() ); } @@ -75,62 +48,4 @@ private ShenYuScenarioSpec testDivideHello() { .build()) .build(); } - - private ShenYuScenarioSpec testRocketMQHello() { - return ShenYuScenarioSpec.builder() - .name("testRocketMQHello") - .beforeEachSpec( - ShenYuBeforeEachSpec.builder() - .addSelectorAndRule( - newSelectorBuilder("selector", Plugin.LOGGING_ROCKETMQ) - .name("1") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build(), - newRuleBuilder("rule") - .name("1") - .matchMode(MatchMode.OR) - .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) - .build() - ) - .checker(exists(TEST)) - .build() - ) - .caseSpec( - ShenYuCaseSpec.builder() - .add(request -> { - AtomicBoolean isLog = new AtomicBoolean(false); - try { - Thread.sleep(1000 * 30); - request.request(Method.GET, "/http/order/findById?id=23"); - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMERGROUP); - consumer.setNamesrvAddr(NAMESERVER); - consumer.subscribe(TOPIC, "*"); - consumer.registerMessageListener((MessageListenerConcurrently) (msgs, consumeConcurrentlyContext) -> { - LOG.info("Msg:{}", msgs); - if (CollectionUtils.isNotEmpty(msgs)) { - msgs.forEach(e -> { - if (new String(e.getBody()).contains("/http/order/findById?id=23")) { - isLog.set(true); - } - }); - } - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - }); - LOG.info("consumer.start ; isLog.get():{}", isLog.get()); - consumer.start(); - Thread.sleep(1000 * 30); - LOG.info("isLog.get():{}", isLog.get()); - Assertions.assertTrue(isLog.get()); - } catch (Exception e) { - LOG.error("error", e); - Assertions.assertTrue(isLog.get()); - } - }) - .build() - ) -// .afterEachSpec(ShenYuAfterEachSpec.builder() -// .deleteWaiting(notExists(TEST)).build()) - .build(); - } } diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 6bbe62f37e9f..83e9af82dac1 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -105,17 +105,6 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); - LOG.info("start loggingRocketMQ plugin"); - Map reqBody = new HashMap<>(); - reqBody.put("pluginId", "29"); - reqBody.put("name", "loggingRocketMQ"); - reqBody.put("enabled", "true"); - reqBody.put("role", "Logging"); - reqBody.put("sort", "170"); - reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); - reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); - adminClient.changePluginStatus("1801816010882822166", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); } @ShenYuScenario(provider = DividePluginCases.class) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh similarity index 97% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 725064463c7d..55f770d3c876 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-kafka-sync-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -37,7 +37,6 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health -# docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh new file mode 100644 index 000000000000..f28dc8a83e62 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# init kubernetes for mysql +SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")") +bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh + +# init register center +CUR_PATH=$(readlink -f "$(dirname "$0")") +PRGDIR=$(dirname "$CUR_PATH") +# init shenyu sync +SYNC_ARRAY=("websocket" "http" "zookeeper" "etcd") +#SYNC_ARRAY=("websocket" "nacos") +#MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") + +docker network create -d bridge shenyu + +for sync in "${SYNC_ARRAY[@]}"; do + echo -e "------------------\n" + echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml up -d --quiet-pull + sleep 30s + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull + sleep 30s + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health + sleep 10s + docker ps -a + ## run e2e-test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-kafka -am test + # shellcheck disable=SC2181 + if (($?)); then + echo "${sync}-sync-e2e-test failed" + echo "------------------" + echo "shenyu-admin log:" + echo "------------------" + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml logs shenyu-admin + echo "shenyu-bootstrap log:" + echo "------------------" + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml logs shenyu-bootstrap + echo "shenyu-rocketmq log:" + echo "------------------" + docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml logs + exit 1 + fi + docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml down + docker compose -f "${PRGDIR}"/shenyu-rocketmq-compose.yml down + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml down + echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " +done diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-examples-http-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-examples-http-compose.yml new file mode 100644 index 000000000000..ce8c8a7e7b22 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-examples-http-compose.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.9' + +services: + shenyu-examples-http: + image: shenyu-examples-http:latest + container_name: shenyu-examples-http + environment: + - shenyu.register.serverLists=http://shenyu-admin:9095 + ports: + - "31189:8189" + healthcheck: + test: [ "CMD-SHELL", "wget -q -O - http://localhost:8189/actuator/health | grep UP || exit 1" ] + interval: 10s + timeout: 2s + retries: 3 + start_period: 10s + restart: always + networks: + - shenyu + +networks: + shenyu: + name: shenyu + driver: bridge + external: true diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-rocketmq-compose.yml similarity index 100% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-rocketmq-compose.yml rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/shenyu-rocketmq-compose.yml diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh new file mode 100644 index 000000000000..07650c592b8a --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +docker save shenyu-examples-http:latest | sudo k3s ctr images import - + +# init kubernetes for mysql +SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")") +bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh + +# init register center +CUR_PATH=$(readlink -f "$(dirname "$0")") +PRGDIR=$(dirname "$CUR_PATH") +kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-cm.yml + +# init shenyu sync +SYNC_ARRAY=("websocket" "http" "zookeeper" "etcd") +#SYNC_ARRAY=("websocket" "nacos") +MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") +for sync in ${SYNC_ARRAY[@]}; do + echo -e "------------------\n" + kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health + + sleep 30s + echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" + # shellcheck disable=SC2199 + # shellcheck disable=SC2076 + # shellcheck disable=SC2154 + if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml + sleep 10s + fi + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health + kubectl apply -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health + sleep 10s + kubectl get pod -o wide + + kubectl logs "$(kubectl get pod -o wide | grep shenyu-admin | awk '{print $1}')" + + ## run e2e-test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-kafka -am test + # shellcheck disable=SC2181 + if (($?)); then + echo "${sync}-sync-e2e-test failed" + echo "shenyu-admin log:" + echo "------------------" + kubectl logs "$(kubectl get pod -o wide | grep shenyu-admin | awk '{print $1}')" + echo "shenyu-bootstrap log:" + echo "------------------" + kubectl logs "$(kubectl get pod -o wide | grep shenyu-bootstrap | awk '{print $1}')" + exit 1 + fi + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-mysql.yml + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-admin-"${sync}".yml + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/sync/shenyu-bootstrap-"${sync}".yml + kubectl delete -f "${PRGDIR}"/shenyu-kafka.yml + + # shellcheck disable=SC2199 + # shellcheck disable=SC2076 + if [[ "${MIDDLEWARE_SYNC_ARRAY[@]}" =~ "${sync}" ]]; then + kubectl delete -f "${SHENYU_TESTCASE_DIR}"/k8s/shenyu-"${sync}".yml + fi + echo "[Remove ${sync} synchronous] delete shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" +done diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml new file mode 100644 index 000000000000..0d95d1759997 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + namespace: default + labels: + app: zookeeper +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + image: zookeeper:3.7 + ports: + - containerPort: 2181 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: default + labels: + app: zookeeper +spec: + ports: + - port: 2181 + name: client + selector: + app: zookeeper + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka + namespace: default + labels: + app: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + spec: + containers: + - name: kafka + image: bitnami/kafka:3.6.2 + env: + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://kafka:9092 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zookeeper:2181 + ports: + - containerPort: 9092 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka + namespace: default + labels: + app: kafka +spec: + type: NodePort + ports: + - port: 9092 + name: client + protocol: TCP + targetPort: 9092 + nodePort: 31877 + selector: + app: kafka \ No newline at end of file diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/pom.xml new file mode 100644 index 000000000000..f9e8eef96397 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/pom.xml @@ -0,0 +1,36 @@ + + + + + org.apache.shenyu + shenyu-e2e-case + 0.0.1-SNAPSHOT + + 4.0.0 + shenyu-e2e-case-logging-rocketmq + + + + org.apache.rocketmq + rocketmq-client + 4.9.3 + + + diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java new file mode 100644 index 000000000000..1482843eae65 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.logging.rocketmq; + +import org.apache.shenyu.e2e.client.WaitDataSync; +import org.apache.shenyu.e2e.client.admin.AdminClient; +import org.apache.shenyu.e2e.client.gateway.GatewayClient; +import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; +import org.apache.shenyu.e2e.enums.ServiceTypeEnum; +import org.junit.jupiter.api.Test; + +/** + * Testing the correctness of etcd data synchronization method. + */ +@ShenYuTest(environments = { + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-admin", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31095", + type = ServiceTypeEnum.SHENYU_ADMIN, + parameters = { + @ShenYuTest.Parameter(key = "username", value = "admin"), + @ShenYuTest.Parameter(key = "password", value = "123456") + } + ) + ), + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-gateway", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:31195", + type = ServiceTypeEnum.SHENYU_GATEWAY + ) + ) +}) +public class DataSynTest { + + @Test + void testDataSyn(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { + adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + } +} diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginCases.java new file mode 100644 index 000000000000..bc24271d34da --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginCases.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.logging.rocketmq; + +import com.google.common.collect.Lists; +import io.restassured.http.Method; +import org.apache.commons.collections.CollectionUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.shenyu.e2e.engine.scenario.ShenYuScenarioProvider; +import org.apache.shenyu.e2e.engine.scenario.specification.ScenarioSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuBeforeEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuCaseSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.ShenYuScenarioSpec; +import org.apache.shenyu.e2e.model.MatchMode; +import org.apache.shenyu.e2e.model.Plugin; +import org.apache.shenyu.e2e.model.data.Condition; +import org.junit.jupiter.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.shenyu.e2e.engine.scenario.function.HttpCheckers.exists; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newConditions; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newRuleBuilder; +import static org.apache.shenyu.e2e.template.ResourceDataTemplate.newSelectorBuilder; + +public class DividePluginCases implements ShenYuScenarioProvider { + + private static final String NAMESERVER = "http://localhost:31876"; + + private static final String CONSUMERGROUP = "shenyu-plugin-logging-rocketmq"; + + private static final String TOPIC = "shenyu-access-logging"; + + private static final String TEST = "/http/order/findById?id=123"; + + private static final Logger LOG = LoggerFactory.getLogger(DividePluginCases.class); + + @Override + public List get() { + return Lists.newArrayList( + testDivideHello(), + testRocketMQHello() + ); + } + + private ShenYuScenarioSpec testDivideHello() { + return ShenYuScenarioSpec.builder() + .name("http client hello1") + .beforeEachSpec(ShenYuBeforeEachSpec.builder() + .checker(exists("/http/order/findById?id=123")) + .build()) + .caseSpec(ShenYuCaseSpec.builder() + .addExists("/http/order/findById?id=123") + .build()) + .build(); + } + + private ShenYuScenarioSpec testRocketMQHello() { + return ShenYuScenarioSpec.builder() + .name("testRocketMQHello") + .beforeEachSpec( + ShenYuBeforeEachSpec.builder() + .addSelectorAndRule( + newSelectorBuilder("selector", Plugin.LOGGING_ROCKETMQ) + .name("1") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build(), + newRuleBuilder("rule") + .name("1") + .matchMode(MatchMode.OR) + .conditionList(newConditions(Condition.ParamType.URI, Condition.Operator.STARTS_WITH, "/http")) + .build() + ) + .checker(exists(TEST)) + .build() + ) + .caseSpec( + ShenYuCaseSpec.builder() + .add(request -> { + AtomicBoolean isLog = new AtomicBoolean(false); + try { + Thread.sleep(1000 * 30); + request.request(Method.GET, "/http/order/findById?id=23"); + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMERGROUP); + consumer.setNamesrvAddr(NAMESERVER); + consumer.subscribe(TOPIC, "*"); + consumer.registerMessageListener((MessageListenerConcurrently) (msgs, consumeConcurrentlyContext) -> { + LOG.info("Msg:{}", msgs); + if (CollectionUtils.isNotEmpty(msgs)) { + msgs.forEach(e -> { + if (new String(e.getBody()).contains("/http/order/findById?id=23")) { + isLog.set(true); + } + }); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + }); + LOG.info("consumer.start ; isLog.get():{}", isLog.get()); + consumer.start(); + Thread.sleep(1000 * 30); + LOG.info("isLog.get():{}", isLog.get()); + Assertions.assertTrue(isLog.get()); + } catch (Exception e) { + LOG.error("error", e); + Assertions.assertTrue(isLog.get()); + } + }) + .build() + ) +// .afterEachSpec(ShenYuAfterEachSpec.builder() +// .deleteWaiting(notExists(TEST)).build()) + .build(); + } +} diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java new file mode 100644 index 000000000000..1cd2c9bc1e39 --- /dev/null +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.e2e.testcase.logging.rocketmq; + +import com.google.common.collect.Lists; +import org.apache.shenyu.e2e.client.WaitDataSync; +import org.apache.shenyu.e2e.client.admin.AdminClient; +import org.apache.shenyu.e2e.client.gateway.GatewayClient; +import org.apache.shenyu.e2e.constant.Constants; +import org.apache.shenyu.e2e.engine.annotation.ShenYuScenario; +import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; +import org.apache.shenyu.e2e.engine.scenario.specification.AfterEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.BeforeEachSpec; +import org.apache.shenyu.e2e.engine.scenario.specification.CaseSpec; +import org.apache.shenyu.e2e.enums.ServiceTypeEnum; +import org.apache.shenyu.e2e.model.ResourcesData; +import org.apache.shenyu.e2e.model.data.BindingData; +import org.apache.shenyu.e2e.model.response.SelectorDTO; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.shenyu.e2e.constant.Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID; + +@ShenYuTest(environments = { + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-admin", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:9095", + type = ServiceTypeEnum.SHENYU_ADMIN, + parameters = { + @ShenYuTest.Parameter(key = "username", value = "admin"), + @ShenYuTest.Parameter(key = "password", value = "123456") + } + ) + ), + @ShenYuTest.Environment( + serviceName = "shenyu-e2e-gateway", + service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", + baseUrl = "http://localhost:9195", + type = ServiceTypeEnum.SHENYU_GATEWAY + ) + ) +}) +public class DividePluginTest { + + private static final Logger LOG = LoggerFactory.getLogger(DividePluginTest.class); + + private List selectorIds = Lists.newArrayList(); + + @BeforeEach + void before(final AdminClient client, final GatewayClient gateway, final BeforeEachSpec spec) { + spec.getChecker().check(gateway); + + ResourcesData resources = spec.getResources(); + for (ResourcesData.Resource res : resources.getResources()) { + SelectorDTO dto = client.create(res.getSelector()); + selectorIds.add(dto.getId()); + res.getRules().forEach(rule -> { + rule.setSelectorId(dto.getId()); + client.create(rule); + }); + BindingData bindingData = res.getBindingData(); + if (Objects.nonNull(bindingData)) { + bindingData.setSelectorId(dto.getId()); + bindingData.setNamespaceId(SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + client.bindingData(bindingData); + } + } + + spec.getWaiting().waitFor(gateway); + } + + @AfterEach + void after(final AdminClient client, final GatewayClient gateway, final AfterEachSpec spec) { + spec.getDeleter().delete(client, selectorIds); + spec.deleteWaiting().waitFor(gateway); + selectorIds = Lists.newArrayList(); + } + + @BeforeAll + void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { + adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + LOG.info("start loggingRocketMQ plugin"); + Map reqBody = new HashMap<>(); + reqBody.put("pluginId", "29"); + reqBody.put("name", "loggingRocketMQ"); + reqBody.put("enabled", "true"); + reqBody.put("role", "Logging"); + reqBody.put("sort", "170"); + reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); + reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); + adminClient.changePluginStatus("1801816010882822166", reqBody); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); + } + + @ShenYuScenario(provider = DividePluginCases.class) + void testDivide(final GatewayClient gateway, final CaseSpec spec) { + spec.getVerifiers().forEach(verifier -> verifier.verify(gateway.getHttpRequesterSupplier().get())); + } +} From 6d48a4fd5ac35ca76dfbe6650c86782dbf33e0dd Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 11:53:50 +0800 Subject: [PATCH 065/155] checkstyle --- .../org/apache/shenyu/e2e/testcase/http/DividePluginTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java index 83e9af82dac1..b66fb619b3f4 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/src/test/java/org/apache/shenyu/e2e/testcase/http/DividePluginTest.java @@ -21,7 +21,6 @@ import org.apache.shenyu.e2e.client.WaitDataSync; import org.apache.shenyu.e2e.client.admin.AdminClient; import org.apache.shenyu.e2e.client.gateway.GatewayClient; -import org.apache.shenyu.e2e.constant.Constants; import org.apache.shenyu.e2e.engine.annotation.ShenYuScenario; import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; import org.apache.shenyu.e2e.engine.scenario.specification.BeforeEachSpec; @@ -35,9 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import static org.apache.shenyu.e2e.constant.Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID; From 941ae4e3fc30e5173d7b92cfb989da75b94c7b65 Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 12:34:30 +0800 Subject: [PATCH 066/155] e2e port --- .../apache/shenyu/e2e/testcase/kafka/DividePluginTest.java | 4 ++-- .../e2e/testcase/logging/rocketmq/DividePluginTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index cb7e9a84c6de..cc750706c0ba 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9095", + baseUrl = "http://localhost:31095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9195", + baseUrl = "http://localhost:31195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java index 1cd2c9bc1e39..a7abd95c9e67 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9095", + baseUrl = "http://localhost:31095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9195", + baseUrl = "http://localhost:31195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) From c1ba974636a8005748861f71321d329dadbabe75 Mon Sep 17 00:00:00 2001 From: liuhy Date: Fri, 6 Dec 2024 12:54:24 +0800 Subject: [PATCH 067/155] e2e kafka logging --- .../compose/shenyu-kafka-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 05fe00fe7ca9..458aa8b8cf7e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -35,7 +35,7 @@ services: - shenyu kafka: - image: bitnami/kafka:latest + image: wurstmeister/kafka:latest container_name: shenyu-kafka ports: - "31877:9092" @@ -51,9 +51,9 @@ services: restart: always environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:31877 - KAFKA_LISTENERS: PLAINTEXT://shenyu-kafka:31877 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 depends_on: zookeeper: condition: service_healthy From 76eecec4a947e123c880d2005b5ec7b51fb16a9d Mon Sep 17 00:00:00 2001 From: liuhy Date: Sun, 8 Dec 2024 11:06:30 +0800 Subject: [PATCH 068/155] e2e kafka logging --- .../shenyu/e2e/testcase/kafka/DividePluginTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java index cc750706c0ba..c4ca80f8ed73 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java @@ -103,9 +103,7 @@ void after(final AdminClient client, final GatewayClient gateway, final AfterEac @BeforeAll void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { adminClient.login(); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + Map reqBody = new HashMap<>(); LOG.info("start loggingKafka plugin"); reqBody.put("pluginId", "33"); @@ -118,6 +116,10 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); } @ShenYuScenario(provider = DividePluginCases.class) From f405cc227f4434e1eac68a22c78dd922c859a40d Mon Sep 17 00:00:00 2001 From: liuhy Date: Sun, 8 Dec 2024 11:06:55 +0800 Subject: [PATCH 069/155] e2e rocketmq logging --- .../e2e/testcase/logging/rocketmq/DividePluginTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java index a7abd95c9e67..4841b2235a58 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java @@ -103,9 +103,7 @@ void after(final AdminClient client, final GatewayClient gateway, final AfterEac @BeforeAll void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { adminClient.login(); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + LOG.info("start loggingRocketMQ plugin"); Map reqBody = new HashMap<>(); reqBody.put("pluginId", "29"); @@ -117,6 +115,11 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); + + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + } @ShenYuScenario(provider = DividePluginCases.class) From 007aa34073d9c6e681246d42d33d90ad15bfedd5 Mon Sep 17 00:00:00 2001 From: liuhy Date: Sun, 8 Dec 2024 15:37:12 +0800 Subject: [PATCH 070/155] e2e rocketmq logging --- .../compose/script/e2e-logging-rocketmq-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh index f28dc8a83e62..f62d9b33d182 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/compose/script/e2e-logging-rocketmq-compose.sh @@ -44,7 +44,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 10s docker ps -a ## run e2e-test - ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-kafka -am test + ./mvnw -B -f ./shenyu-e2e/pom.xml -pl shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq -am test # shellcheck disable=SC2181 if (($?)); then echo "${sync}-sync-e2e-test failed" From aeaffeda659c6368e6848c7cd4d226b377cd96b3 Mon Sep 17 00:00:00 2001 From: liuhy Date: Sun, 8 Dec 2024 15:53:14 +0800 Subject: [PATCH 071/155] e2e rocketmq logging --- .../e2e/testcase/logging/rocketmq/DividePluginTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java index 4841b2235a58..4961cc3b8af9 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java @@ -104,6 +104,10 @@ void after(final AdminClient client, final GatewayClient gateway, final AfterEac void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + LOG.info("start loggingRocketMQ plugin"); Map reqBody = new HashMap<>(); reqBody.put("pluginId", "29"); @@ -116,10 +120,6 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr adminClient.changePluginStatus("1801816010882822166", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); - } @ShenYuScenario(provider = DividePluginCases.class) From 8bf49d1fe831ab9b98c5c1173ab7292f070b134f Mon Sep 17 00:00:00 2001 From: liuhy Date: Sun, 8 Dec 2024 16:17:53 +0800 Subject: [PATCH 072/155] e2e rocketmq logging --- .../e2e/testcase/kafka/DataSynTest.java | 59 ------------------- .../logging/rocketmq/DataSynTest.java | 59 ------------------- 2 files changed, 118 deletions(-) delete mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java delete mode 100644 shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java deleted file mode 100644 index 9b8262834139..000000000000 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DataSynTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.shenyu.e2e.testcase.kafka; - -import org.apache.shenyu.e2e.client.WaitDataSync; -import org.apache.shenyu.e2e.client.admin.AdminClient; -import org.apache.shenyu.e2e.client.gateway.GatewayClient; -import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; -import org.apache.shenyu.e2e.enums.ServiceTypeEnum; -import org.junit.jupiter.api.Test; - -/** - * Testing the correctness of etcd data synchronization method. - */ -@ShenYuTest(environments = { - @ShenYuTest.Environment( - serviceName = "shenyu-e2e-admin", - service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31095", - type = ServiceTypeEnum.SHENYU_ADMIN, - parameters = { - @ShenYuTest.Parameter(key = "username", value = "admin"), - @ShenYuTest.Parameter(key = "password", value = "123456") - } - ) - ), - @ShenYuTest.Environment( - serviceName = "shenyu-e2e-gateway", - service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31195", - type = ServiceTypeEnum.SHENYU_GATEWAY - ) - ) -}) -public class DataSynTest { - - @Test - void testDataSyn(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { - adminClient.login(); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); - } -} diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java deleted file mode 100644 index 1482843eae65..000000000000 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DataSynTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.shenyu.e2e.testcase.logging.rocketmq; - -import org.apache.shenyu.e2e.client.WaitDataSync; -import org.apache.shenyu.e2e.client.admin.AdminClient; -import org.apache.shenyu.e2e.client.gateway.GatewayClient; -import org.apache.shenyu.e2e.engine.annotation.ShenYuTest; -import org.apache.shenyu.e2e.enums.ServiceTypeEnum; -import org.junit.jupiter.api.Test; - -/** - * Testing the correctness of etcd data synchronization method. - */ -@ShenYuTest(environments = { - @ShenYuTest.Environment( - serviceName = "shenyu-e2e-admin", - service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31095", - type = ServiceTypeEnum.SHENYU_ADMIN, - parameters = { - @ShenYuTest.Parameter(key = "username", value = "admin"), - @ShenYuTest.Parameter(key = "password", value = "123456") - } - ) - ), - @ShenYuTest.Environment( - serviceName = "shenyu-e2e-gateway", - service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31195", - type = ServiceTypeEnum.SHENYU_GATEWAY - ) - ) -}) -public class DataSynTest { - - @Test - void testDataSyn(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { - adminClient.login(); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); - } -} From 0725fd93873eb7daee45f7642493b0c5a8c861a2 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 10:58:14 +0800 Subject: [PATCH 073/155] e2e rocketmq logging --- .../e2e/testcase/{ => logging}/kafka/DividePluginCases.java | 2 +- .../e2e/testcase/{ => logging}/kafka/DividePluginTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/{ => logging}/kafka/DividePluginCases.java (99%) rename shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/{ => logging}/kafka/DividePluginTest.java (99%) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java similarity index 99% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 4d1adcd2a20e..23ccf43f02d3 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shenyu.e2e.testcase.kafka; +package org.apache.shenyu.e2e.testcase.logging.kafka; import com.google.common.collect.Lists; import io.restassured.http.Method; diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java similarity index 99% rename from shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java rename to shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index c4ca80f8ed73..fe21dbf1c788 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.shenyu.e2e.testcase.kafka; +package org.apache.shenyu.e2e.testcase.logging.kafka; import com.google.common.collect.Lists; import org.apache.shenyu.e2e.client.WaitDataSync; From 78362a19ea4c497b70e4c908bf689b59804a68f6 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 11:19:41 +0800 Subject: [PATCH 074/155] e2e rocketmq logging --- .../e2e/testcase/logging/kafka/DividePluginTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index fe21dbf1c788..d70c1f47034a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -104,6 +104,10 @@ void after(final AdminClient client, final GatewayClient gateway, final AfterEac void setup(final AdminClient adminClient, final GatewayClient gatewayClient) throws Exception { adminClient.login(); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); + WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); + Map reqBody = new HashMap<>(); LOG.info("start loggingKafka plugin"); reqBody.put("pluginId", "33"); @@ -116,10 +120,6 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); - - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllSelectors, gatewayClient::getSelectorCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllMetaData, gatewayClient::getMetaDataCache, adminClient); - WaitDataSync.waitAdmin2GatewayDataSyncEquals(adminClient::listAllRules, gatewayClient::getRuleCache, adminClient); } @ShenYuScenario(provider = DividePluginCases.class) From 1c27c879c48206d38e43f220ff9e3ba7103351b1 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 11:30:51 +0800 Subject: [PATCH 075/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 4 +++- .../e2e/testcase/logging/rocketmq/DividePluginTest.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index d70c1f47034a..6e1afc4d7f90 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -119,7 +119,9 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("config", "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka"); + Map plugins = gatewayClient.getPlugins(); + LOG.info("shenyu e2e plugin list ={}", plugins); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.kafka.LoggingKafkaPlugin"); } @ShenYuScenario(provider = DividePluginCases.class) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java index 4961cc3b8af9..66f6ee5bbe27 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/src/test/java/org/apache/shenyu/e2e/testcase/logging/rocketmq/DividePluginTest.java @@ -118,7 +118,9 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", "{\"topic\":\"shenyu-access-logging\", \"namesrvAddr\": \"rocketmq-dialevoneid:9876\",\"producerGroup\":\"shenyu-plugin-logging-rocketmq\"}"); adminClient.changePluginStatus("1801816010882822166", reqBody); - WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq"); + Map plugins = gatewayClient.getPlugins(); + LOG.info("shenyu e2e plugin list ={}", plugins); + WaitDataSync.waitGatewayPluginUse(gatewayClient, "org.apache.shenyu.plugin.logging.rocketmq.LoggingRocketMQPlugin"); } From a11edd6cbf9b6a17803fec04a20e00a7c2164a63 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 15:14:40 +0800 Subject: [PATCH 076/155] e2e rocketmq logging --- .../main/java/org/apache/shenyu/e2e/client/WaitDataSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java index 172b7b987a36..95f5ecf79716 100644 --- a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java +++ b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/WaitDataSync.java @@ -87,7 +87,7 @@ public static void waitGatewayPluginUse(final GatewayClient gatewayClient, final LOGGER.info("pluginMap:{}", MAPPER.writeValueAsString(pluginMap)); int retryNum = 0; boolean existPlugin = false; - while (!existPlugin && retryNum < 5) { + while (!existPlugin && retryNum < 10) { for (String plugin : pluginMap.keySet()) { if (plugin.startsWith(pluginClass)) { existPlugin = true; From 63423a9c4afb253187360f83897404086cd8d348 Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 19:21:31 +0800 Subject: [PATCH 077/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 458aa8b8cf7e..8cd9b1ccb875 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -51,8 +51,8 @@ services: restart: always environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 depends_on: zookeeper: From 7cb46b03dc420abb4b6905c9875c4e8e5cdaadcd Mon Sep 17 00:00:00 2001 From: liuhy Date: Mon, 9 Dec 2024 19:42:57 +0800 Subject: [PATCH 078/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 8cd9b1ccb875..8f433c8e1041 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -52,8 +52,8 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://0.0.0.0:31877 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:31877 depends_on: zookeeper: condition: service_healthy From c1646bf578dc86c48d1c440b9c3c57e4b723075a Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 09:27:30 +0800 Subject: [PATCH 079/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 8f433c8e1041..dc720948e01a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -52,8 +52,8 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://0.0.0.0:31877 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:31877 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 + KAFKA_LISTENERS: PLAINTEXT://shenyu-kafka:9092 depends_on: zookeeper: condition: service_healthy From 067e4d9e1afaeba2ff45b5b1f768333ef7892106 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 09:59:09 +0800 Subject: [PATCH 080/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index dc720948e01a..1b1507fd760f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -20,8 +20,8 @@ services: image: bitnami/zookeeper:latest container_name: shenyu-zookeeper environment: - ZOO_STANDALONE_ENABLED: "true" - ALLOW_ANONYMOUS_LOGIN: "yes" + - ZOO_STANDALONE_ENABLED="true" + - ALLOW_ANONYMOUS_LOGIN="yes" ports: - "31878:2181" healthcheck: @@ -35,7 +35,7 @@ services: - shenyu kafka: - image: wurstmeister/kafka:latest + image: bitnami/kafka:3.4.0 container_name: shenyu-kafka ports: - "31877:9092" @@ -50,10 +50,12 @@ services: # condition: service_healthy restart: always environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: shenyu-zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://shenyu-kafka:9092 + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_LISTENERS=PLAINTEXT://:9092 depends_on: zookeeper: condition: service_healthy From a6995147c511c5a887ee63195bfa64423095abb2 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 10:00:10 +0800 Subject: [PATCH 081/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 1b1507fd760f..8944baca508b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -18,7 +18,7 @@ services: zookeeper: image: bitnami/zookeeper:latest - container_name: shenyu-zookeeper + container_name: zookeeper environment: - ZOO_STANDALONE_ENABLED="true" - ALLOW_ANONYMOUS_LOGIN="yes" @@ -36,7 +36,7 @@ services: kafka: image: bitnami/kafka:3.4.0 - container_name: shenyu-kafka + container_name: kafka ports: - "31877:9092" # healthcheck: From 5fa778c13aeefecabe052234ec2ddfe419a1135e Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 10:19:30 +0800 Subject: [PATCH 082/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 8944baca508b..7919f554e151 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -20,8 +20,8 @@ services: image: bitnami/zookeeper:latest container_name: zookeeper environment: - - ZOO_STANDALONE_ENABLED="true" - - ALLOW_ANONYMOUS_LOGIN="yes" + - ZOO_STANDALONE_ENABLED=true + - ALLOW_ANONYMOUS_LOGIN=yes ports: - "31878:2181" healthcheck: From 93fe110a9a963d4451a0f43c126d0bf613deb521 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 14:59:47 +0800 Subject: [PATCH 083/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 12 ++++++------ .../testcase/logging/kafka/DividePluginCases.java | 6 +++--- .../e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- .../shenyu/e2e/client/gateway/GatewayClient.java | 4 ++++ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 7919f554e151..299dd5542458 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -23,7 +23,7 @@ services: - ZOO_STANDALONE_ENABLED=true - ALLOW_ANONYMOUS_LOGIN=yes ports: - - "31878:2181" + - "2181:2181" healthcheck: test: [ "CMD-SHELL", "nc -z localhost 2181 || exit 1" ] interval: 30s @@ -35,10 +35,11 @@ services: - shenyu kafka: - image: bitnami/kafka:3.4.0 + image: wurstmeister/kafka container_name: kafka + hostname: kafka ports: - - "31877:9092" + - "9092:9092" # healthcheck: # test: [ "CMD-SHELL", "wget -q -O - http://localhost:9092/actuator/health | grep UP || exit 1" ] # interval: 10s @@ -51,11 +52,10 @@ services: restart: always environment: - KAFKA_BROKER_ID=1 + - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 + - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes - - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 - - KAFKA_LISTENERS=PLAINTEXT://:9092 depends_on: zookeeper: condition: service_healthy diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 23ccf43f02d3..4e6b1d35e085 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -66,10 +66,10 @@ private ShenYuScenarioSpec testDivideHello() { return ShenYuScenarioSpec.builder() .name("http client hello1") .beforeEachSpec(ShenYuBeforeEachSpec.builder() - .checker(exists("/http/order/findById?id=123")) + .checker(exists(TEST)) .build()) .caseSpec(ShenYuCaseSpec.builder() - .addExists("/http/order/findById?id=123") + .addExists(TEST) .build()) .build(); } @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:31877"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); AtomicReference keepCosuming = new AtomicReference<>(true); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 6e1afc4d7f90..9f562d9e314c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:31877\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); diff --git a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/gateway/GatewayClient.java b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/gateway/GatewayClient.java index 5ffd1a61eb3a..50afe58dd725 100644 --- a/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/gateway/GatewayClient.java +++ b/shenyu-e2e/shenyu-e2e-client/src/main/java/org/apache/shenyu/e2e/client/gateway/GatewayClient.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.restassured.response.Response; import io.restassured.specification.RequestSpecification; +import org.apache.commons.collections4.CollectionUtils; import org.apache.shenyu.e2e.annotation.ShenYuGatewayClient; import org.apache.shenyu.e2e.client.BaseClient; import org.apache.shenyu.e2e.common.RequestLogConsumer; @@ -178,6 +179,9 @@ public List getSelectorCache() throws JsonProcessingException List selectorDataList = new ArrayList<>(); for (Map.Entry entry : s.entrySet()) { List list = (List) entry.getValue(); + if (CollectionUtils.isEmpty(list)) { + continue; + } String json = MAPPER.writeValueAsString(list.get(0)); SelectorCacheData selectorData = MAPPER.readValue(json, SelectorCacheData.class); selectorDataList.add(selectorData); From aa313f83fc6cada56ca2faee74b1a4164d867df6 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 15:22:27 +0800 Subject: [PATCH 084/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 4e6b1d35e085..de378b5c9f57 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -119,7 +119,8 @@ private ShenYuScenarioSpec testKafkaHello() { ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); records.forEach(record -> { String message = record.value(); - if (message.contains("/http/order/findById?id=23")) { + LOG.info("kafka message:{}", message); + if (message.contains("/http/order/findById")) { isLog.set(true); keepCosuming.set(false); } From 4a01c9d2e44144dbf1f61f5fbbfbc46398143c91 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 15:44:53 +0800 Subject: [PATCH 085/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index de378b5c9f57..1a84fa2cd66f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -113,7 +113,7 @@ private ShenYuScenarioSpec testKafkaHello() { AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); while (keepCosuming.get()) { - if (Duration.between(start, Instant.now()).toMillis() > 60000) { + if (Duration.between(start, Instant.now()).toMillis() > 150000) { keepCosuming.set(false); } ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); From 54eafb2e52f6e123ee2012767459e2f02afd17c1 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 16:07:30 +0800 Subject: [PATCH 086/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 1a84fa2cd66f..b7c12b6228dc 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -113,7 +113,7 @@ private ShenYuScenarioSpec testKafkaHello() { AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); while (keepCosuming.get()) { - if (Duration.between(start, Instant.now()).toMillis() > 150000) { + if (Duration.between(start, Instant.now()).toMillis() > 300000) { keepCosuming.set(false); } ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); From 9dd39dde9b5f10540dfc63b28ad58cfd22034d6f Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 16:10:24 +0800 Subject: [PATCH 087/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index b7c12b6228dc..23d3cc3a1dde 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -110,6 +110,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); + Thread.sleep(1000 * 30); AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); while (keepCosuming.get()) { From fc53725fa61776de4dd9481f9e68f1a9908740dc Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 17:11:46 +0800 Subject: [PATCH 088/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 23d3cc3a1dde..ac9210956b7d 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://kafka:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 9f562d9e314c..510c7a843ada 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From bbc9da035b52968a0892ec3c6a22870d456e2ede Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 17:39:27 +0800 Subject: [PATCH 089/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 299dd5542458..b78455536963 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -52,7 +52,7 @@ services: restart: always environment: - KAFKA_BROKER_ID=1 - - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 + - KAFKA_ADVERTISED_HOST_NAME=kafka - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes From 283a8bbc27c81b7dd2843efb7db68956460a79d2 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 18:01:57 +0800 Subject: [PATCH 090/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index b78455536963..7bc328e8ea82 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -55,6 +55,7 @@ services: - KAFKA_ADVERTISED_HOST_NAME=kafka - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENERS=PLAINTEXT://kafka:9092 - ALLOW_PLAINTEXT_LISTENER=yes depends_on: zookeeper: From bf596baa3756bc8607e134c9ec5cff59f0ec1f52 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 18:33:54 +0800 Subject: [PATCH 091/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index ac9210956b7d..23d3cc3a1dde 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://kafka:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); From 73ab4cf1f19680bcdfc0d2930602b029d5298d7c Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 18:59:28 +0800 Subject: [PATCH 092/155] e2e rocketmq logging --- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 510c7a843ada..9f562d9e314c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From c27591a8dcf5b65e942859baba6ef5d0a95888f5 Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 19:51:50 +0800 Subject: [PATCH 093/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 7bc328e8ea82..95e67c9a02f8 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -52,10 +52,10 @@ services: restart: always environment: - KAFKA_BROKER_ID=1 - - KAFKA_ADVERTISED_HOST_NAME=kafka + - KAFKA_ADVERTISED_HOST_NAME=localhost - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 - ALLOW_PLAINTEXT_LISTENER=yes depends_on: zookeeper: From ef46ca87e01fa4b824a516cef1982220f0a7ebcb Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 19:55:56 +0800 Subject: [PATCH 094/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 95e67c9a02f8..a0ef4fafe55b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -56,7 +56,6 @@ services: - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 - - ALLOW_PLAINTEXT_LISTENER=yes depends_on: zookeeper: condition: service_healthy From d3a6c08f2c02e3552fbc6b938fcb84e2903dbd3c Mon Sep 17 00:00:00 2001 From: liuhy Date: Tue, 10 Dec 2024 20:00:06 +0800 Subject: [PATCH 095/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index a0ef4fafe55b..1c9fabbaffd7 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -61,6 +61,7 @@ services: condition: service_healthy networks: - shenyu + - host networks: shenyu: From 407be063cb70ae6a7a792527c0bc0d7b4ec98971 Mon Sep 17 00:00:00 2001 From: liuhy Date: Wed, 11 Dec 2024 09:21:39 +0800 Subject: [PATCH 096/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 1c9fabbaffd7..605eb9ded121 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -38,6 +38,7 @@ services: image: wurstmeister/kafka container_name: kafka hostname: kafka + network_mode: host ports: - "9092:9092" # healthcheck: @@ -59,9 +60,6 @@ services: depends_on: zookeeper: condition: service_healthy - networks: - - shenyu - - host networks: shenyu: From be102a7118e55f70f44492904252b5a6b7a5d79b Mon Sep 17 00:00:00 2001 From: liuhy Date: Wed, 11 Dec 2024 09:46:01 +0800 Subject: [PATCH 097/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 605eb9ded121..a6fde8b8cb34 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -19,6 +19,7 @@ services: zookeeper: image: bitnami/zookeeper:latest container_name: zookeeper + network_mode: host environment: - ZOO_STANDALONE_ENABLED=true - ALLOW_ANONYMOUS_LOGIN=yes @@ -31,8 +32,6 @@ services: retries: 5 start_period: 40s restart: always - networks: - - shenyu kafka: image: wurstmeister/kafka @@ -55,15 +54,9 @@ services: - KAFKA_BROKER_ID=1 - KAFKA_ADVERTISED_HOST_NAME=localhost - KAFKA_ADVERTISED_PORT=9092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 + - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 + - KAFKA_LISTENERS=PLAINTEXT://localhost:9092 depends_on: zookeeper: condition: service_healthy -networks: - shenyu: - name: shenyu - driver: bridge - external: true - From 6b2c2ecea5b2c23c8cc8ca8f47df15d303eb93ca Mon Sep 17 00:00:00 2001 From: liuhy Date: Wed, 11 Dec 2024 10:41:35 +0800 Subject: [PATCH 098/155] e2e rocketmq logging --- .../compose/shenyu-kafka-compose.yml | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index a6fde8b8cb34..be6a891967c7 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -23,8 +23,6 @@ services: environment: - ZOO_STANDALONE_ENABLED=true - ALLOW_ANONYMOUS_LOGIN=yes - ports: - - "2181:2181" healthcheck: test: [ "CMD-SHELL", "nc -z localhost 2181 || exit 1" ] interval: 30s @@ -34,28 +32,15 @@ services: restart: always kafka: - image: wurstmeister/kafka + image: wurstmeister/kafka:latest container_name: kafka - hostname: kafka network_mode: host - ports: - - "9092:9092" -# healthcheck: -# test: [ "CMD-SHELL", "wget -q -O - http://localhost:9092/actuator/health | grep UP || exit 1" ] -# interval: 10s -# timeout: 2s -# retries: 3 -# start_period: 10s -# depends_on: -# zookeeper: -# condition: service_healthy restart: always environment: - KAFKA_BROKER_ID=1 - - KAFKA_ADVERTISED_HOST_NAME=localhost - - KAFKA_ADVERTISED_PORT=9092 - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - - KAFKA_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 depends_on: zookeeper: condition: service_healthy From 291135f0e82eca03694cf286c301ecee1b9d4737 Mon Sep 17 00:00:00 2001 From: liuhy Date: Wed, 11 Dec 2024 17:23:47 +0800 Subject: [PATCH 099/155] e2e rocketmq logging --- .github/workflows/e2e-k8s.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index bb57c2dfed10..753a9e530def 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -220,8 +220,8 @@ jobs: script: e2e-cluster-zookeeper-compose - case: shenyu-e2e-case-logging-rocketmq script: e2e-logging-rocketmq-compose - - case: shenyu-e2e-case-logging-kafka - script: e2e-logging-kafka-compose +# - case: shenyu-e2e-case-logging-kafka +# script: e2e-logging-kafka-compose steps: - uses: actions/checkout@v2 From e763b2925b1a285188bd771c23a48792ecd306bd Mon Sep 17 00:00:00 2001 From: liuhy Date: Wed, 11 Dec 2024 18:23:11 +0800 Subject: [PATCH 100/155] e2e rocketmq logging --- .github/workflows/e2e-k8s.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 753a9e530def..bb57c2dfed10 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -220,8 +220,8 @@ jobs: script: e2e-cluster-zookeeper-compose - case: shenyu-e2e-case-logging-rocketmq script: e2e-logging-rocketmq-compose -# - case: shenyu-e2e-case-logging-kafka -# script: e2e-logging-kafka-compose + - case: shenyu-e2e-case-logging-kafka + script: e2e-logging-kafka-compose steps: - uses: actions/checkout@v2 From 2da57facc0127c4687be0e77c525c0ff5c95c484 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 14 Dec 2024 12:37:49 +0800 Subject: [PATCH 101/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index be6a891967c7..a41fbcb4c3fb 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -41,6 +41,8 @@ services: - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 + ports: + - 9092:9092 depends_on: zookeeper: condition: service_healthy From ef01ab6d404f8f63e6d8e9b6f1138bf23a367aeb Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 14 Dec 2024 17:16:37 +0800 Subject: [PATCH 102/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index a41fbcb4c3fb..56b711f15a3c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -42,7 +42,7 @@ services: - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 ports: - - 9092:9092 + - "31877:9092" depends_on: zookeeper: condition: service_healthy diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 23d3cc3a1dde..4d73f208157e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:31877"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); From f416abcd64c1af7ff0ad9ec13cd0de6404a37d72 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 21:30:32 +0800 Subject: [PATCH 103/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 56b711f15a3c..65eeda2fa47c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,8 +39,6 @@ services: environment: - KAFKA_BROKER_ID=1 - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 ports: - "31877:9092" depends_on: From b8fe08f1c97e4cf17adc8b4863d242ee000fc076 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 21:52:11 +0800 Subject: [PATCH 104/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 4d73f208157e..a0bac22616c1 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -57,8 +57,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { @Override public List get() { return Lists.newArrayList( - testDivideHello(), - testKafkaHello() + testDivideHello() +// testKafkaHello() ); } From 72a4794efbde9de44e85dd55d94de5b480ebe2a5 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 22:29:06 +0800 Subject: [PATCH 105/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index a0bac22616c1..a6f9594aa75f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -58,7 +58,7 @@ public class DividePluginCases implements ShenYuScenarioProvider { public List get() { return Lists.newArrayList( testDivideHello() -// testKafkaHello() + testKafkaHello() ); } From b3a0e8cdadfda59efe1eebf368a83292068d63d9 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 22:45:16 +0800 Subject: [PATCH 106/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index a6f9594aa75f..4d73f208157e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -57,7 +57,7 @@ public class DividePluginCases implements ShenYuScenarioProvider { @Override public List get() { return Lists.newArrayList( - testDivideHello() + testDivideHello(), testKafkaHello() ); } From d7d5f3a1fe1cc4c2c1ccec3a7aa02e9a97611b43 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 23:35:19 +0800 Subject: [PATCH 107/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 65eeda2fa47c..56cd823726a3 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,6 +39,8 @@ services: environment: - KAFKA_BROKER_ID=1 - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_LISTENERS=PLAINTEXT://localhost:9092 ports: - "31877:9092" depends_on: From bdbb0aabfc4abc0b76639604a419ea760e5ae101 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 15 Dec 2024 23:37:41 +0800 Subject: [PATCH 108/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 56cd823726a3..036d128074e9 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,7 +39,7 @@ services: environment: - KAFKA_BROKER_ID=1 - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:31877 - KAFKA_LISTENERS=PLAINTEXT://localhost:9092 ports: - "31877:9092" From 2abe0c868a6e9a1edd56b75691b5a1ec99bc9e10 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 16 Dec 2024 08:55:47 +0800 Subject: [PATCH 109/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 036d128074e9..efbd718b7b99 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -40,9 +40,9 @@ services: - KAFKA_BROKER_ID=1 - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:31877 - - KAFKA_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_LISTENERS=PLAINTEXT://localhost:31877 ports: - - "31877:9092" + - "31877:31877" depends_on: zookeeper: condition: service_healthy From 05d5e4a479d31435974a0082b47cd8d45ec5f695 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 16 Dec 2024 23:06:24 +0800 Subject: [PATCH 110/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml index 0d95d1759997..e440d61bc61f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml @@ -75,11 +75,11 @@ spec: image: bitnami/kafka:3.6.2 env: - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:9092 + value: PLAINTEXT://kafka:31877 - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 ports: - - containerPort: 9092 + - containerPort: 31877 --- apiVersion: v1 From be992d763d82176fdb8d4e788c8ded01c4b26a8c Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 16 Dec 2024 23:10:13 +0800 Subject: [PATCH 111/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml | 4 ++-- .../shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml index e440d61bc61f..4e6831d09545 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml @@ -92,10 +92,10 @@ metadata: spec: type: NodePort ports: - - port: 9092 + - port: 31877 name: client protocol: TCP - targetPort: 9092 + targetPort: 31877 nodePort: 31877 selector: app: kafka \ No newline at end of file diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml index 0d95d1759997..c26323fd4e49 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml @@ -75,7 +75,7 @@ spec: image: bitnami/kafka:3.6.2 env: - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:9092 + value: PLAINTEXT://kafka:31877 - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 ports: From 6d4bcce9df766ae8f886e5d738a56bb3a3ebe252 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 16 Dec 2024 23:37:31 +0800 Subject: [PATCH 112/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml | 6 +++--- .../shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml index c26323fd4e49..4e6831d09545 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml @@ -79,7 +79,7 @@ spec: - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 ports: - - containerPort: 9092 + - containerPort: 31877 --- apiVersion: v1 @@ -92,10 +92,10 @@ metadata: spec: type: NodePort ports: - - port: 9092 + - port: 31877 name: client protocol: TCP - targetPort: 9092 + targetPort: 31877 nodePort: 31877 selector: app: kafka \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java b/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java index 74ff33eb5fe5..35b628ab33f2 100644 --- a/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java +++ b/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java @@ -72,7 +72,7 @@ public void setUp() { this.requestLog = new ShenyuRequestLog(); MockServerHttpRequest request = MockServerHttpRequest .get("localhost") - .remoteAddress(new InetSocketAddress(9092)) + .remoteAddress(new InetSocketAddress(31877)) .header("X-source", "mock test") .queryParam("queryParam", "Hello,World") .build(); From 51836ab23b352393f42193dc3851305f77c429a4 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 17 Dec 2024 22:54:31 +0800 Subject: [PATCH 113/155] [type:feat]add kafka logging e2e test --- .../script/e2e-logging-kafka-compose.sh | 2 - .../compose/shenyu-kafka-compose.yml | 84 ++++++++++++++----- .../k8s/script/e2e-http-sync.sh | 2 +- .../k8s/shenyu-kafka.yml | 10 +-- .../logging/kafka/DividePluginCases.java | 2 +- .../k8s/script/e2e-http-sync.sh | 2 +- .../k8s/shenyu-kafka.yml | 2 +- .../logging/kafka/LoggingKafkaPluginTest.java | 2 +- 8 files changed, 71 insertions(+), 35 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 55f770d3c876..0d04581d651f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -37,8 +37,6 @@ for sync in "${SYNC_ARRAY[@]}"; do sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull - sleep 30s docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index efbd718b7b99..e7ee41b4eabd 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -16,34 +16,72 @@ services: - zookeeper: - image: bitnami/zookeeper:latest - container_name: zookeeper - network_mode: host - environment: - - ZOO_STANDALONE_ENABLED=true - - ALLOW_ANONYMOUS_LOGIN=yes - healthcheck: - test: [ "CMD-SHELL", "nc -z localhost 2181 || exit 1" ] - interval: 30s - timeout: 10s - retries: 5 - start_period: 40s + shenyu-zk: + container_name: shenyu-zk + image: zookeeper:latest restart: always + networks: + - shenyu - kafka: - image: wurstmeister/kafka:latest - container_name: kafka - network_mode: host + shenyu-examples-http: + deploy: + resources: + limits: + memory: 2048M + container_name: shenyu-examples-http + image: shenyu-examples-http:latest restart: always environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=localhost:2181 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:31877 - - KAFKA_LISTENERS=PLAINTEXT://localhost:31877 + - shenyu.register.serverLists=http://shenyu-admin:9095 + healthcheck: + test: [ "CMD", "wget", "http://shenyu-examples-http:8189/test/path/123?name=tom" ] + timeout: 2s + retries: 30 ports: - - "31877:31877" + - "8189:8189" + depends_on: + shenyu-integrated-test-http: + condition: service_healthy + networks: + - shenyu + + shenyu-admin: + image: apache/shenyu-admin:latest + container_name: shenyu-admin + restart: always + networks: + - shenyu depends_on: - zookeeper: + shenyu-redis: condition: service_healthy + ports: + - "9095:9095" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - shenyu.database.init_script=sql-script/h2/schema.sql + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://shenyu-admin:9095/actuator/health | grep UP || exit 1"] + timeout: 2s + retries: 30 + + shenyu-kafka: + container_name: shenyu-kafka + image: confluentinc/cp-kafka:latest + restart: always + depends_on: + - shenyu-zk + networks: + - shenyu + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: shenyu-zk + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 +networks: + shenyu: + name: shenyu \ No newline at end of file diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh index 07650c592b8a..2ca65f63ddaf 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/script/e2e-http-sync.sh @@ -34,7 +34,7 @@ MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") for sync in ${SYNC_ARRAY[@]}; do echo -e "------------------\n" kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml - sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:9092/actuator/health sleep 30s echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml index 4e6831d09545..9c5c05fad911 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/k8s/shenyu-kafka.yml @@ -75,11 +75,11 @@ spec: image: bitnami/kafka:3.6.2 env: - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:31877 + value: PLAINTEXT://kafka:9092 - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 ports: - - containerPort: 31877 + - containerPort: 9092 --- apiVersion: v1 @@ -92,10 +92,10 @@ metadata: spec: type: NodePort ports: - - port: 31877 + - port: 9092 name: client protocol: TCP - targetPort: 31877 - nodePort: 31877 + targetPort: 9092 + nodePort: 9092 selector: app: kafka \ No newline at end of file diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 4d73f208157e..23d3cc3a1dde 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:31877"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh index 07650c592b8a..2ca65f63ddaf 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/script/e2e-http-sync.sh @@ -34,7 +34,7 @@ MIDDLEWARE_SYNC_ARRAY=("zookeeper" "etcd" "nacos") for sync in ${SYNC_ARRAY[@]}; do echo -e "------------------\n" kubectl apply -f "${PRGDIR}"/shenyu-kafka.yml - sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31877/actuator/health + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:9092/actuator/health sleep 30s echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml shenyu-examples-springcloud.yml" diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml index 4e6831d09545..94a63c58ae47 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/k8s/shenyu-kafka.yml @@ -75,7 +75,7 @@ spec: image: bitnami/kafka:3.6.2 env: - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:31877 + value: PLAINTEXT://kafka:9092 - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 ports: diff --git a/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java b/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java index 35b628ab33f2..74ff33eb5fe5 100644 --- a/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java +++ b/shenyu-plugin/shenyu-plugin-logging/shenyu-plugin-logging-kafka/src/test/java/org/apache/shenyu/plugin/logging/kafka/LoggingKafkaPluginTest.java @@ -72,7 +72,7 @@ public void setUp() { this.requestLog = new ShenyuRequestLog(); MockServerHttpRequest request = MockServerHttpRequest .get("localhost") - .remoteAddress(new InetSocketAddress(31877)) + .remoteAddress(new InetSocketAddress(9092)) .header("X-source", "mock test") .queryParam("queryParam", "Hello,World") .build(); From 6e8afa54910913a4ec4d3942771d2748c64a3fa2 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 17 Dec 2024 23:23:28 +0800 Subject: [PATCH 114/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index e7ee41b4eabd..dd4f7e296944 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,9 +39,6 @@ services: retries: 30 ports: - "8189:8189" - depends_on: - shenyu-integrated-test-http: - condition: service_healthy networks: - shenyu @@ -51,9 +48,6 @@ services: restart: always networks: - shenyu - depends_on: - shenyu-redis: - condition: service_healthy ports: - "9095:9095" environment: From 82df2a8d0dd63b70c1c0f8084e5b9d44000794ee Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 6 Jan 2025 22:38:05 +0800 Subject: [PATCH 115/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index bb57c2dfed10..753a9e530def 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -220,8 +220,8 @@ jobs: script: e2e-cluster-zookeeper-compose - case: shenyu-e2e-case-logging-rocketmq script: e2e-logging-rocketmq-compose - - case: shenyu-e2e-case-logging-kafka - script: e2e-logging-kafka-compose +# - case: shenyu-e2e-case-logging-kafka +# script: e2e-logging-kafka-compose steps: - uses: actions/checkout@v2 From 4687aa932f18fbaccc831ae8bfd483418cfa9a01 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 7 Jan 2025 22:54:55 +0800 Subject: [PATCH 116/155] [type:feat]add kafka logging e2e test --- .../main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java index 88bc0ddcd0eb..ffd9a5406882 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java @@ -22,7 +22,7 @@ import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; /** - * shenyu admin start. + * shenyu admin start */ @SpringBootApplication(exclude = {LdapAutoConfiguration.class}) public class ShenyuAdminBootstrap { From 3b78ffae0fa85aa0b5c7ae4c5cd32a42d29b22e2 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 8 Jan 2025 09:17:01 +0800 Subject: [PATCH 117/155] [type:feat]add kafka logging e2e test --- .../main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java index ffd9a5406882..796164adcf02 100644 --- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/ShenyuAdminBootstrap.java @@ -22,7 +22,7 @@ import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; /** - * shenyu admin start + * shenyu admin startShenyuAdminBootstrap. */ @SpringBootApplication(exclude = {LdapAutoConfiguration.class}) public class ShenyuAdminBootstrap { From ae0ec7b366014c808e7d06a649b1ee070275e884 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 8 Jan 2025 22:13:35 +0800 Subject: [PATCH 118/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 753a9e530def..bb57c2dfed10 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -220,8 +220,8 @@ jobs: script: e2e-cluster-zookeeper-compose - case: shenyu-e2e-case-logging-rocketmq script: e2e-logging-rocketmq-compose -# - case: shenyu-e2e-case-logging-kafka -# script: e2e-logging-kafka-compose + - case: shenyu-e2e-case-logging-kafka + script: e2e-logging-kafka-compose steps: - uses: actions/checkout@v2 From 56f227505263885456b1a55947ef6d05bf88d4c1 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 8 Jan 2025 22:46:19 +0800 Subject: [PATCH 119/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-logging-kafka-compose.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 0d04581d651f..7aa1001ae756 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -38,6 +38,7 @@ for sync in "${SYNC_ARRAY[@]}"; do sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health sleep 10s From 7a8f53ee2cad3e00aad7ec8b72d20912f3323e71 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 10:25:43 +0800 Subject: [PATCH 120/155] [type:feat]add kafka logging e2e test --- .../e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 23d3cc3a1dde..3dbd94a86488 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "http://localhost:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 9f562d9e314c..fa061bf1b043 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31095", + baseUrl = "http://localhost:9095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31195", + baseUrl = "http://localhost:9195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From de271a21e554b10a26d54dddb2b73aa7088dcbdf Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 13:56:22 +0800 Subject: [PATCH 121/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 3dbd94a86488..84655955f56a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -109,7 +109,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); - consumer.subscribe(Arrays.asList(TOPIC)); + consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index fa061bf1b043..d68d92266c6f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9095", + baseUrl = "http://localhost:31095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9195", + baseUrl = "http://localhost:31195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) From b43d13ee03c4973911dc85aa1c2f0c7e64956f4b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 14:31:56 +0800 Subject: [PATCH 122/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 84655955f56a..3dbd94a86488 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -109,7 +109,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); - consumer.subscribe(Arrays.asList(TOPIC)); + consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); AtomicReference keepCosuming = new AtomicReference<>(true); Instant start = Instant.now(); From 54103142c492b374c2dc625e67495e27ba372de3 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 16:25:56 +0800 Subject: [PATCH 123/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index dd4f7e296944..0f7ceddcf0ad 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -71,7 +71,7 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-docker:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 3dbd94a86488..26b493ad07fa 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka-docker:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index d68d92266c6f..81fe78407d31 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka-docker:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From 7a46aa2d8bc62da3d62ce6d103bb4c93ee949950 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 17:10:34 +0800 Subject: [PATCH 124/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 0f7ceddcf0ad..62a927c3a6c6 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -71,7 +71,7 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-docker:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 26b493ad07fa..a77a0d71e63f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -107,7 +107,7 @@ private ShenYuScenarioSpec testKafkaHello() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka-docker:9092"); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "shenyu-kafka:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 81fe78407d31..49cfae4ea14c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"kafka-docker:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"shenyu-kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From 71eb84126abcb7949318cf575464b8397a8b4552 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 19:46:52 +0800 Subject: [PATCH 125/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 62a927c3a6c6..d1b0dfea85bc 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -78,4 +78,6 @@ services: networks: shenyu: - name: shenyu \ No newline at end of file + name: shenyu + driver: bridge + external: true \ No newline at end of file From bb24971eeecc0767a74fe2aa2f6f12c3d0512702 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 12 Jan 2025 23:42:21 +0800 Subject: [PATCH 126/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 52 +++++++------------ 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index d1b0dfea85bc..4f00a3453ec1 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -23,6 +23,24 @@ services: networks: - shenyu + shenyu-kafka: + container_name: shenyu-kafka + image: confluentinc/cp-kafka:latest + restart: always + depends_on: + - shenyu-zk + networks: + - shenyu + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: shenyu-zk + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + shenyu-examples-http: deploy: resources: @@ -42,40 +60,6 @@ services: networks: - shenyu - shenyu-admin: - image: apache/shenyu-admin:latest - container_name: shenyu-admin - restart: always - networks: - - shenyu - ports: - - "9095:9095" - environment: - - SPRING_PROFILES_ACTIVE=h2 - - shenyu.database.init_script=sql-script/h2/schema.sql - healthcheck: - test: ["CMD-SHELL", "wget -q -O - http://shenyu-admin:9095/actuator/health | grep UP || exit 1"] - timeout: 2s - retries: 30 - - shenyu-kafka: - container_name: shenyu-kafka - image: confluentinc/cp-kafka:latest - restart: always - depends_on: - - shenyu-zk - networks: - - shenyu - ports: - - "9092:9092" - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: shenyu-zk - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - networks: shenyu: name: shenyu From 25e6e677eab26d870c97287fda8346fa74cf3589 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 13 Jan 2025 22:32:56 +0800 Subject: [PATCH 127/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index a77a0d71e63f..4df10f337f37 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -57,8 +57,7 @@ public class DividePluginCases implements ShenYuScenarioProvider { @Override public List get() { return Lists.newArrayList( - testDivideHello(), - testKafkaHello() + testDivideHello() ); } From c3f166f7d66ae305152a9e08a40732a9c3c03f42 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 14 Jan 2025 22:38:32 +0800 Subject: [PATCH 128/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 49cfae4ea14c..2c743aa3063e 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31095", + baseUrl = "http://localhost:9095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:31195", + baseUrl = "http://localhost:9195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) From acff20d467c2441115940d31a866b5c9da6f3a89 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 14 Jan 2025 23:04:16 +0800 Subject: [PATCH 129/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 4 ++-- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 4df10f337f37..d6b58a527591 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -57,7 +57,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { @Override public List get() { return Lists.newArrayList( - testDivideHello() + testDivideHello(), + testKafkaHello() ); } @@ -105,7 +106,6 @@ private ShenYuScenarioSpec testKafkaHello() { StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "testGroup"); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "shenyu-kafka:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 2c743aa3063e..49cfae4ea14c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -48,7 +48,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-admin", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9095", + baseUrl = "http://localhost:31095", type = ServiceTypeEnum.SHENYU_ADMIN, parameters = { @ShenYuTest.Parameter(key = "username", value = "admin"), @@ -59,7 +59,7 @@ @ShenYuTest.Environment( serviceName = "shenyu-e2e-gateway", service = @ShenYuTest.ServiceConfigure(moduleName = "shenyu-e2e", - baseUrl = "http://localhost:9195", + baseUrl = "http://localhost:31195", type = ServiceTypeEnum.SHENYU_GATEWAY ) ) From e3b98a0a88aa1673e281ec9d8aecfa43f07e4ffa Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 18 Jan 2025 13:24:01 +0800 Subject: [PATCH 130/155] [type:feat]add kafka logging e2e test --- .../shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml index 72162ec5ed9d..85c1d967c7ac 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml @@ -30,7 +30,7 @@ org.apache.kafka kafka-clients - 2.4.1 + 3.7.1 From af6ee6127a295185e85038f05ead4617ecdf61eb Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 18 Jan 2025 19:00:43 +0800 Subject: [PATCH 131/155] [type:feat]add kafka logging e2e test --- shenyu-e2e/pom.xml | 1 + .../shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/pom.xml b/shenyu-e2e/pom.xml index 98fcecf3a073..e524324212c7 100644 --- a/shenyu-e2e/pom.xml +++ b/shenyu-e2e/pom.xml @@ -58,6 +58,7 @@ 32.0.0-jre 4.4 1.5.1 + 3.7.1 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml index 85c1d967c7ac..90ca689ce6fb 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/pom.xml @@ -30,7 +30,7 @@ org.apache.kafka kafka-clients - 3.7.1 + ${kafka-clients.version} From 642cdf3997e22524d4c437a02119fa90f7f6e44b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 19 Jan 2025 13:06:15 +0800 Subject: [PATCH 132/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index d6b58a527591..fa3ea18d7046 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -127,7 +127,7 @@ private ShenYuScenarioSpec testKafkaHello() { }); } Assertions.assertTrue(isLog.get()); - } catch (InterruptedException e) { + } catch (Exception e) { LOG.info("isLog.get():{}", isLog.get()); LOG.error("error", e); throw new RuntimeException(e); From 88c0c20784ec17aa982a91728981588c100bda01 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 19 Jan 2025 20:31:35 +0800 Subject: [PATCH 133/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index fa3ea18d7046..11a1cbc3dd5a 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -106,6 +106,7 @@ private ShenYuScenarioSpec testKafkaHello() { StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "shenyu-kafka:9092"); KafkaConsumer consumer = new KafkaConsumer<>(props); consumer.subscribe(Arrays.asList(TOPIC)); @@ -127,7 +128,7 @@ private ShenYuScenarioSpec testKafkaHello() { }); } Assertions.assertTrue(isLog.get()); - } catch (Exception e) { + } catch (InterruptedException e) { LOG.info("isLog.get():{}", isLog.get()); LOG.error("error", e); throw new RuntimeException(e); From 26ffd1dddb4473842e65f33277671a75a145f75c Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 19 Jan 2025 21:23:05 +0800 Subject: [PATCH 134/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- .../testcase/logging/kafka/DividePluginCases.java | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 4f00a3453ec1..c78fcf3ed8da 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -32,7 +32,7 @@ services: networks: - shenyu ports: - - "9092:9092" + - "31092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 11a1cbc3dd5a..0d7748924a0c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -101,14 +101,12 @@ private ShenYuScenarioSpec testKafkaHello() { try { Thread.sleep(1000 * 30); request.request(Method.GET, "/http/order/findById?id=23"); - Properties props = new Properties(); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "shenyu-kafka:9092"); - KafkaConsumer consumer = new KafkaConsumer<>(props); + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:31092"); // Kafka 服务器地址 + properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); // 消费者组ID + properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + KafkaConsumer consumer = new KafkaConsumer<>(properties); consumer.subscribe(Arrays.asList(TOPIC)); Thread.sleep(1000 * 30); AtomicReference keepCosuming = new AtomicReference<>(true); From 8a715ae929af8c10389a23c389ccaf2bba417707 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 19 Jan 2025 21:48:52 +0800 Subject: [PATCH 135/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 0d7748924a0c..d01bed39a4d7 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -102,8 +102,8 @@ private ShenYuScenarioSpec testKafkaHello() { Thread.sleep(1000 * 30); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:31092"); // Kafka 服务器地址 - properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); // 消费者组ID + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:31092"); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); KafkaConsumer consumer = new KafkaConsumer<>(properties); From b626c85c2c17da12ef0cf81246fffe12716f108f Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 19 Jan 2025 22:41:24 +0800 Subject: [PATCH 136/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index 49cfae4ea14c..e96c025c2ee4 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"shenyu-kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"localhost:31092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From 329c85962899ff3284022b4dbf8c2b274fc8228a Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 25 Jan 2025 10:10:26 +0800 Subject: [PATCH 137/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index c78fcf3ed8da..1eb50657c187 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -36,7 +36,7 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://shenyu-kafka:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:31092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 From 2559fe4f9381bdaa6abe8f9bc424a0c1c8eeb540 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 25 Jan 2025 11:21:56 +0800 Subject: [PATCH 138/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 1eb50657c187..2d68cb20ef92 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -36,6 +36,7 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk + listeners: PLAINTEXT://0:0:0:0:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:31092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT From ca00595099378f24a32ba4d4375f4894ca4a66c3 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 25 Jan 2025 12:48:16 +0800 Subject: [PATCH 139/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-logging-kafka-compose.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 7aa1001ae756..074012bc3141 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -35,11 +35,11 @@ for sync in "${SYNC_ARRAY[@]}"; do echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml up -d --quiet-pull sleep 30s - sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull sleep 30s + sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health sleep 10s docker ps -a From c07e4224be3ca0b0235614a63837c352ee884a99 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Fri, 7 Feb 2025 16:45:59 +0800 Subject: [PATCH 140/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-logging-kafka-compose.sh | 3 +++ .../compose/shenyu-kafka-compose.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 074012bc3141..1d049ec797c6 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -20,6 +20,9 @@ SHENYU_TESTCASE_DIR=$(dirname "$(dirname "$(dirname "$(dirname "$0")")")") bash "${SHENYU_TESTCASE_DIR}"/k8s/script/storage/storage_init_mysql.sh +# init ip +export HOST_IP=$(hostname -I | awk '{print $1}') + # init register center CUR_PATH=$(readlink -f "$(dirname "$0")") PRGDIR=$(dirname "$CUR_PATH") diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 2d68cb20ef92..29bd17f8e3bd 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -37,7 +37,7 @@ services: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk listeners: PLAINTEXT://0:0:0:0:9092 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:31092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${HOST_IP}:31092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 From 5304c5cbbaa77f0bc1c45e60c862c7556883c0df Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Fri, 7 Feb 2025 17:45:18 +0800 Subject: [PATCH 141/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index bb57c2dfed10..65303d621dc9 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -96,7 +96,7 @@ jobs: docker save apache/shenyu-admin:latest > /tmp/apache-shenyu-admin.tar - name: Upload Docker Image Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: shenyu-images path: /tmp/apache-shenyu-*.tar From a204506b330826fc62c129e463404890eac7aada Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Fri, 7 Feb 2025 17:56:18 +0800 Subject: [PATCH 142/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 65303d621dc9..9ebb0cd13810 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -75,7 +75,7 @@ jobs: - name: Save ShenYu Maven Repos uses: actions/cache/save@v3 with: - path: ~/.m2/repository + paths: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -87,7 +87,7 @@ jobs: # if: steps.restore-e2e-maven-cache.outputs.cache-hit != 'true' # uses: actions/cache/save@v3 # with: -# path: ~/.m2/repository +# paths: ~/.m2/repository # key: ${{ runner.os }}-e2e-maven-${{ hashFiles('**/pom.xml') }} - name: Build Docker Image @@ -99,7 +99,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: shenyu-images - path: /tmp/apache-shenyu-*.tar + paths: /tmp/apache-shenyu-*.tar retention-days: 1 e2e-storage: @@ -160,7 +160,7 @@ jobs: if: steps.filter.outputs.changed == 'true' uses: actions/cache/restore@v3 with: - path: ~/.m2/repository + paths: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -168,7 +168,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - path: /tmp/shenyu/ + paths: /tmp/shenyu/ - name: Load ShenYu Docker Images run: | @@ -264,7 +264,7 @@ jobs: if: steps.filter.outputs.changed == 'true' uses: actions/cache/restore@v3 with: - path: ~/.m2/repository + paths: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -272,12 +272,12 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - path: /tmp/shenyu/ + paths: /tmp/shenyu/ # - name: Restore ShenYu E2E Maven Repos # uses: actions/cache/restore@v3 # with: -# path: ~/.m2/repository +# paths: ~/.m2/repository # key: ${{ runner.os }}-e2e-maven-${{ hashFiles('**/pom.xml') }} # restore-keys: | # ${{ runner.os }}-e2e-maven- @@ -288,7 +288,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - path: /tmp/shenyu/ + paths: /tmp/shenyu/ - name: Load ShenYu Docker Images run: | From 43ac8809691e7797d6fd5205f12e35856fa91b1e Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Fri, 7 Feb 2025 18:04:11 +0800 Subject: [PATCH 143/155] [type:feat]add kafka logging e2e test --- .github/workflows/e2e-k8s.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 9ebb0cd13810..bb57c2dfed10 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -75,7 +75,7 @@ jobs: - name: Save ShenYu Maven Repos uses: actions/cache/save@v3 with: - paths: ~/.m2/repository + path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -87,7 +87,7 @@ jobs: # if: steps.restore-e2e-maven-cache.outputs.cache-hit != 'true' # uses: actions/cache/save@v3 # with: -# paths: ~/.m2/repository +# path: ~/.m2/repository # key: ${{ runner.os }}-e2e-maven-${{ hashFiles('**/pom.xml') }} - name: Build Docker Image @@ -96,10 +96,10 @@ jobs: docker save apache/shenyu-admin:latest > /tmp/apache-shenyu-admin.tar - name: Upload Docker Image Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: shenyu-images - paths: /tmp/apache-shenyu-*.tar + path: /tmp/apache-shenyu-*.tar retention-days: 1 e2e-storage: @@ -160,7 +160,7 @@ jobs: if: steps.filter.outputs.changed == 'true' uses: actions/cache/restore@v3 with: - paths: ~/.m2/repository + path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -168,7 +168,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - paths: /tmp/shenyu/ + path: /tmp/shenyu/ - name: Load ShenYu Docker Images run: | @@ -264,7 +264,7 @@ jobs: if: steps.filter.outputs.changed == 'true' uses: actions/cache/restore@v3 with: - paths: ~/.m2/repository + path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- @@ -272,12 +272,12 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - paths: /tmp/shenyu/ + path: /tmp/shenyu/ # - name: Restore ShenYu E2E Maven Repos # uses: actions/cache/restore@v3 # with: -# paths: ~/.m2/repository +# path: ~/.m2/repository # key: ${{ runner.os }}-e2e-maven-${{ hashFiles('**/pom.xml') }} # restore-keys: | # ${{ runner.os }}-e2e-maven- @@ -288,7 +288,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: shenyu-images - paths: /tmp/shenyu/ + path: /tmp/shenyu/ - name: Load ShenYu Docker Images run: | From b343f572cc74cb1ed311901a96711d95c66e4364 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 8 Feb 2025 19:59:30 +0800 Subject: [PATCH 144/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-logging-kafka-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index 1d049ec797c6..b51cb5a4a248 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -39,8 +39,8 @@ for sync in "${SYNC_ARRAY[@]}"; do docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull + docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31189/actuator/health From 1b97dd6bf3dd63707b07560709b3d6ac13d0de66 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 8 Feb 2025 20:20:20 +0800 Subject: [PATCH 145/155] [type:feat]add kafka logging e2e test --- .../compose/script/e2e-logging-kafka-compose.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh index b51cb5a4a248..bef0307c66cd 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/script/e2e-logging-kafka-compose.sh @@ -35,11 +35,12 @@ docker network create -d bridge shenyu for sync in "${SYNC_ARRAY[@]}"; do echo -e "------------------\n" + docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull + sleep 30s echo "[Start ${sync} synchronous] create shenyu-admin-${sync}.yml shenyu-bootstrap-${sync}.yml " docker compose -f "$SHENYU_TESTCASE_DIR"/compose/sync/shenyu-sync-"${sync}".yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31195/actuator/health - docker compose -f "${PRGDIR}"/shenyu-kafka-compose.yml up -d --quiet-pull docker compose -f "${PRGDIR}"/shenyu-examples-http-compose.yml up -d --quiet-pull sleep 30s sh "$SHENYU_TESTCASE_DIR"/k8s/script/healthcheck.sh http://localhost:31095/actuator/health From 9be8358b95c968050e8012ff8095065de3671ed4 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 8 Feb 2025 21:19:51 +0800 Subject: [PATCH 146/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- .../shenyu/e2e/testcase/logging/kafka/DividePluginTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index d01bed39a4d7..d4dbcd5395e4 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -102,7 +102,7 @@ private ShenYuScenarioSpec testKafkaHello() { Thread.sleep(1000 * 30); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:31092"); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java index e96c025c2ee4..49cfae4ea14c 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginTest.java @@ -117,7 +117,7 @@ void setup(final AdminClient adminClient, final GatewayClient gatewayClient) thr reqBody.put("sort", "180"); reqBody.put("namespaceId", Constants.SYS_DEFAULT_NAMESPACE_NAMESPACE_ID); reqBody.put("config", - "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"localhost:31092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); + "{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"shenyu-kafka:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}"); adminClient.changePluginStatus("1801816010882822171", reqBody); Map plugins = gatewayClient.getPlugins(); LOG.info("shenyu e2e plugin list ={}", plugins); From 16f97ddeeb8639efda56dd514356ee3584e5f71b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sat, 8 Feb 2025 22:17:01 +0800 Subject: [PATCH 147/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 29bd17f8e3bd..8c125fa8aace 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -20,6 +20,8 @@ services: container_name: shenyu-zk image: zookeeper:latest restart: always + ports: + - "2181:2181" networks: - shenyu @@ -32,12 +34,12 @@ services: networks: - shenyu ports: - - "31092:9092" + - "9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk listeners: PLAINTEXT://0:0:0:0:9092 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${HOST_IP}:31092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 From 9c92c44de1e72699d723730d0561aee9c240758c Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 9 Feb 2025 18:12:14 +0800 Subject: [PATCH 148/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index d4dbcd5395e4..cd7e80f3ff0b 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -57,8 +57,8 @@ public class DividePluginCases implements ShenYuScenarioProvider { @Override public List get() { return Lists.newArrayList( - testDivideHello(), - testKafkaHello() + testDivideHello() +// testKafkaHello() ); } From 34ba929082a1a91707d8367a1724a9a367da9094 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Sun, 9 Feb 2025 18:57:46 +0800 Subject: [PATCH 149/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 2 +- .../testcase/logging/kafka/DividePluginCases.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 8c125fa8aace..03b28331eaca 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,7 +39,7 @@ services: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: shenyu-zk listeners: PLAINTEXT://0:0:0:0:9092 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${HOST_IP}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index cd7e80f3ff0b..723390214dca 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -34,6 +34,8 @@ import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -52,13 +54,16 @@ public class DividePluginCases implements ShenYuScenarioProvider { private static final String TEST = "/http/order/findById?id=123"; + @Value("${HOST_IP}") + private static String kafkaBroker; + private static final Logger LOG = LoggerFactory.getLogger(DividePluginCases.class); @Override public List get() { return Lists.newArrayList( - testDivideHello() -// testKafkaHello() + testDivideHello(), + testKafkaHello() ); } @@ -100,9 +105,11 @@ private ShenYuScenarioSpec testKafkaHello() { AtomicBoolean isLog = new AtomicBoolean(false); try { Thread.sleep(1000 * 30); + kafkaBroker = kafkaBroker+":9092"; + LOG.info("kafkaBroker = "+kafkaBroker); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker); properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); From 86fd60ed56c3a35435552b0546d9213306e58996 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 10 Feb 2025 21:09:43 +0800 Subject: [PATCH 150/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 723390214dca..ad0c54eaffe2 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -105,7 +105,7 @@ private ShenYuScenarioSpec testKafkaHello() { AtomicBoolean isLog = new AtomicBoolean(false); try { Thread.sleep(1000 * 30); - kafkaBroker = kafkaBroker+":9092"; + kafkaBroker = kafkaBroker + ":9092"; LOG.info("kafkaBroker = "+kafkaBroker); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); From 91bcd6b7b20c9285baa4e2d7d145922866c7ef7b Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 10 Feb 2025 21:26:34 +0800 Subject: [PATCH 151/155] [type:feat]add kafka logging e2e test --- .../shenyu/e2e/testcase/logging/kafka/DividePluginCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index ad0c54eaffe2..52299912f682 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -106,7 +106,7 @@ private ShenYuScenarioSpec testKafkaHello() { try { Thread.sleep(1000 * 30); kafkaBroker = kafkaBroker + ":9092"; - LOG.info("kafkaBroker = "+kafkaBroker); + LOG.info("kafkaBroker = " + kafkaBroker); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker); From d8ec762265108b4bcd08775f20ab1083fe30766e Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Mon, 10 Feb 2025 21:55:34 +0800 Subject: [PATCH 152/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 03b28331eaca..098bf215e2aa 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -37,11 +37,12 @@ services: - "9092:9092" environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: shenyu-zk + KAFKA_ZOOKEEPER_CONNECT: "shenyu-zk:2181" listeners: PLAINTEXT://0:0:0:0:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${HOST_IP}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 shenyu-examples-http: From 9d22bcfec1748c3bee16d92659b1026c69550229 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Tue, 11 Feb 2025 22:23:54 +0800 Subject: [PATCH 153/155] [type:feat]add kafka logging e2e test --- .../compose/shenyu-kafka-compose.yml | 4 ++-- .../e2e/testcase/logging/kafka/DividePluginCases.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml index 098bf215e2aa..a29451395b6f 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/compose/shenyu-kafka-compose.yml @@ -39,7 +39,7 @@ services: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: "shenyu-zk:2181" listeners: PLAINTEXT://0:0:0:0:9092 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${HOST_IP}:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" @@ -67,5 +67,5 @@ services: networks: shenyu: name: shenyu - driver: bridge + driver: host external: true \ No newline at end of file diff --git a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java index 52299912f682..72a5db670882 100644 --- a/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java +++ b/shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-kafka/src/test/java/org/apache/shenyu/e2e/testcase/logging/kafka/DividePluginCases.java @@ -105,11 +105,11 @@ private ShenYuScenarioSpec testKafkaHello() { AtomicBoolean isLog = new AtomicBoolean(false); try { Thread.sleep(1000 * 30); - kafkaBroker = kafkaBroker + ":9092"; - LOG.info("kafkaBroker = " + kafkaBroker); +// kafkaBroker = kafkaBroker + ":9092"; +// LOG.info("kafkaBroker = " + kafkaBroker); request.request(Method.GET, "/http/order/findById?id=23"); Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-consumer-group"); properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); From 45273a9a04b98e45049021c721e8c91ab0121481 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 12 Feb 2025 22:42:34 +0800 Subject: [PATCH 154/155] [type:feat]add kafka logging e2e test --- db/init/mysql/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/init/mysql/schema.sql b/db/init/mysql/schema.sql index d744628f1123..a1abd9d0ae04 100644 --- a/db/init/mysql/schema.sql +++ b/db/init/mysql/schema.sql @@ -912,7 +912,7 @@ INSERT INTO `plugin` VALUES ('6', 'dubbo', '{\"register\":\"zookeeper://localhos INSERT INTO `plugin` VALUES ('8', 'springCloud', NULL, 'Proxy', 200, 0, '2022-05-25 18:02:53', '2022-05-25 18:02:53',null); INSERT INTO `plugin` VALUES ('9', 'hystrix', NULL, 'FaultTolerance', 130, 0, '2022-05-25 18:02:53', '2022-05-25 18:02:53',null); INSERT INTO `plugin` VALUES ('32', 'loggingElasticSearch','{\"host\":\"localhost\", \"port\": \"9200\"}', 'Logging', 190, 0, '2022-06-19 22:00:00', '2022-06-19 22:00:00',null); -INSERT INTO `plugin` VALUES ('33', 'loggingKafka','{\"host\":\"localhost\", \"port\": \"9092\"}', 'Logging', 180, 0, '2022-07-04 22:00:00', '2022-07-02 22:00:00',null); +INSERT INTO `plugin` VALUES ('33', 'loggingKafka','{\"topic\":\"shenyu-access-logging1\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}', 'Logging', 180, 0, '2022-07-04 22:00:00', '2022-07-02 22:00:00',null); INSERT INTO `plugin` VALUES ('34', 'loggingAliyunSls','{\"projectName\": \"shenyu\", \"logStoreName\": \"shenyu-logstore\", \"topic\": \"shenyu-topic\"}', 'Logging', 175, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null); INSERT INTO `plugin` VALUES ('35', 'loggingPulsar', '{\"topic":\"shenyu-access-logging\", \"serviceUrl\": \"pulsar://localhost:6650\"}', 'Logging', 185, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null); INSERT INTO `plugin` VALUES ('36', 'loggingTencentCls','{\"endpoint\": \"ap-guangzhou.cls.tencentcs.com\", \"topic\": \"shenyu-topic\"}', 'Logging', 176, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null); From 8ce86cf6dd62733d71ba0f31359ed520e04fe6a7 Mon Sep 17 00:00:00 2001 From: huk <2509913103@qq.com> Date: Wed, 12 Feb 2025 23:14:09 +0800 Subject: [PATCH 155/155] [type:feat]add kafka logging e2e test --- db/init/mysql/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/init/mysql/schema.sql b/db/init/mysql/schema.sql index a1abd9d0ae04..04f2fa3be88c 100644 --- a/db/init/mysql/schema.sql +++ b/db/init/mysql/schema.sql @@ -912,7 +912,7 @@ INSERT INTO `plugin` VALUES ('6', 'dubbo', '{\"register\":\"zookeeper://localhos INSERT INTO `plugin` VALUES ('8', 'springCloud', NULL, 'Proxy', 200, 0, '2022-05-25 18:02:53', '2022-05-25 18:02:53',null); INSERT INTO `plugin` VALUES ('9', 'hystrix', NULL, 'FaultTolerance', 130, 0, '2022-05-25 18:02:53', '2022-05-25 18:02:53',null); INSERT INTO `plugin` VALUES ('32', 'loggingElasticSearch','{\"host\":\"localhost\", \"port\": \"9200\"}', 'Logging', 190, 0, '2022-06-19 22:00:00', '2022-06-19 22:00:00',null); -INSERT INTO `plugin` VALUES ('33', 'loggingKafka','{\"topic\":\"shenyu-access-logging1\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}', 'Logging', 180, 0, '2022-07-04 22:00:00', '2022-07-02 22:00:00',null); +INSERT INTO `plugin` VALUES ('33', 'loggingKafka','{\"topic\":\"shenyu-access-logging\",\"namesrvAddr\":\"http://localhost:9092\",\"sampleRate\":\"1\",\"maxResponseBody\":524288,\"maxRequestBody\":524288,\"compressAlg\":\"none\"}', 'Logging', 180, 0, '2022-07-04 22:00:00', '2022-07-02 22:00:00',null); INSERT INTO `plugin` VALUES ('34', 'loggingAliyunSls','{\"projectName\": \"shenyu\", \"logStoreName\": \"shenyu-logstore\", \"topic\": \"shenyu-topic\"}', 'Logging', 175, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null); INSERT INTO `plugin` VALUES ('35', 'loggingPulsar', '{\"topic":\"shenyu-access-logging\", \"serviceUrl\": \"pulsar://localhost:6650\"}', 'Logging', 185, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null); INSERT INTO `plugin` VALUES ('36', 'loggingTencentCls','{\"endpoint\": \"ap-guangzhou.cls.tencentcs.com\", \"topic\": \"shenyu-topic\"}', 'Logging', 176, 0, '2022-06-30 21:00:00', '2022-06-30 21:00:00',null);