improve the http sync strategy.

dongyun624 · Jul 13, 2020 · 5ce842f · 5ce842f
1 parent 8a40c8b
commit 5ce842f
Show file tree

Hide file tree

Showing 21 changed files with 514 additions and 158 deletions.
diff --git a/doc/user-dataSync.md b/doc/user-dataSync.md
@@ -34,12 +34,7 @@ description: 使用不同的数据同步策略
   #urls:是指 soul-admin的地址，如果有多个，请使用（,）分割.
    ```
 
-* soul-admin 配置, 或在 soul-admin 启动参数中设置 `--soul.sync.websocket='' `,然后重启服务。
-```yaml
-soul:
-  sync:
-     websocket:
-```
+* soul-admin 配置, 默认是开启 websocket 同步的，如果您想关闭，请指定`soul.sync.websocket.enabled=false`
 
 * 当建立连接以后会全量获取一次数据，以后的数据都是增量的更新与新增，性能好。
 
@@ -106,19 +101,19 @@ soul:
                url: http://localhost:9095
   #url: 配置成你的 soul-admin的 ip与端口地址，多个admin集群环境请使用（,）分隔。
    ```
-* soul-admin 配置, 或在 soul-admin 启动参数中设置 `--soul.sync.http='' `,然后重启服务。
+* soul-admin 配置, 默认是开启 http 同步的，如果您想关闭，请指定`soul.sync.http.enabled=false`
 ```yaml
 soul:
   sync:
      http:
+       refresh-interval: 5m # 默认5min刷新一次本地缓存
+       enabled: true # 默认启用http同步策略
 ```
 
 * http长轮询使得网关很轻量，时效性略低。 
 
 * 其根据分组key来拉取，如果数据量过大，过多，会有一定的影响。 什么意思呢？就是一个组下面的一个小地方更改，会拉取整个的组数据。
 
-* 在soul-admin 集群时候，可能会有bug。
-
 ## nacos同步
 
 * 网关配置（记得重启）

diff --git a/script/soul_checkstyle.xml b/script/soul_checkstyle.xml
@@ -149,7 +149,9 @@
         <module name="ExplicitInitialization"/>
         <module name="FallThrough"/>
         <module name="IllegalInstantiation"/>
-        <module name="IllegalCatch"/>
+        <module name="IllegalCatch">
+            <property name="illegalClassNames" value="Error,Throwable,java.lang.Error,java.lang.Throwable" />
+        </module>
         <module name="IllegalThrows"/>
         <module name="IllegalType">
             <property name="tokens" value="METHOD_DEF,PARAMETER_DEF,VARIABLE_DEF"/>

diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/config/DataSyncConfiguration.java b/soul-admin/src/main/java/org/dromara/soul/admin/config/DataSyncConfiguration.java
@@ -12,6 +12,7 @@
 import org.dromara.soul.admin.service.SyncDataService;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
@@ -30,17 +31,22 @@ public class DataSyncConfiguration {
      * http long polling(default strategy).
      */
     @Configuration
-    @ConditionalOnMissingBean(DataChangedListener.class)
-    @ConditionalOnProperty(name = "soul.sync.http")
-    @Import(HttpLongPollingDataChangedListener.class)
+    @ConditionalOnProperty(name = "soul.sync.http.enabled", havingValue = "true", matchIfMissing = true)
+    @EnableConfigurationProperties(HttpSyncProperties.class)
     static class HttpLongPollingListener {
+
+        @Bean
+        @ConditionalOnMissingBean(HttpLongPollingDataChangedListener.class)
+        public HttpLongPollingDataChangedListener httpLongPollingDataChangedListener(final HttpSyncProperties httpSyncProperties) {
+            return new HttpLongPollingDataChangedListener(httpSyncProperties);
+        }
+
     }
 
     /**
      * The type Zookeeper listener.
      */
     @Configuration
-    @ConditionalOnMissingBean(DataChangedListener.class)
     @ConditionalOnProperty(prefix = "soul.sync.zookeeper", name = "url")
     @Import(ZookeeperConfiguration.class)
     static class ZookeeperListener {
@@ -52,7 +58,8 @@ static class ZookeeperListener {
          * @return the data changed listener
          */
         @Bean
-        public DataChangedListener dataChangedListener(final ZkClient zkClient) {
+        @ConditionalOnMissingBean(ZookeeperDataChangedListener.class)
+        public DataChangedListener zookeeperDataChangedListener(final ZkClient zkClient) {
             return new ZookeeperDataChangedListener(zkClient);
         }
 
@@ -64,6 +71,7 @@ public DataChangedListener dataChangedListener(final ZkClient zkClient) {
          * @return the zookeeper data init
          */
         @Bean
+        @ConditionalOnMissingBean(ZookeeperDataInit.class)
         public ZookeeperDataInit zookeeperDataInit(final ZkClient zkClient, final SyncDataService syncDataService) {
             return new ZookeeperDataInit(zkClient, syncDataService);
         }
@@ -73,7 +81,6 @@ public ZookeeperDataInit zookeeperDataInit(final ZkClient zkClient, final SyncDa
      * The type Nacos listener.
      */
     @Configuration
-    @ConditionalOnMissingBean(DataChangedListener.class)
     @ConditionalOnProperty(prefix = "soul.sync.nacos", name = "url")
     @Import(NacosConfiguration.class)
     static class NacosListener {
@@ -85,17 +92,17 @@ static class NacosListener {
          * @return the data changed listener
          */
         @Bean
-        public DataChangedListener dataChangedListener(final ConfigService configService) {
+        @ConditionalOnMissingBean(NacosDataChangedListener.class)
+        public DataChangedListener nacosDataChangedListener(final ConfigService configService) {
             return new NacosDataChangedListener(configService);
         }
     }
 
     /**
-     * The WebsocketListener.
+     * The WebsocketListener(default strategy).
      */
     @Configuration
-    @ConditionalOnMissingBean(DataChangedListener.class)
-    @ConditionalOnProperty(name = "soul.sync.websocket")
+    @ConditionalOnProperty(name = "soul.sync.websocket.enabled", havingValue = "true", matchIfMissing = true)
     static class WebsocketListener {
 
         /**
@@ -104,7 +111,8 @@ static class WebsocketListener {
          * @return the data changed listener
          */
         @Bean
-        public DataChangedListener dataChangedListener() {
+        @ConditionalOnMissingBean(WebsocketDataChangedListener.class)
+        public DataChangedListener websocketDataChangedListener() {
             return new WebsocketDataChangedListener();
         }
 
@@ -114,6 +122,7 @@ public DataChangedListener dataChangedListener() {
          * @return the websocket collector
          */
         @Bean
+        @ConditionalOnMissingBean(WebsocketCollector.class)
         public WebsocketCollector websocketCollector() {
             return new WebsocketCollector();
         }
@@ -124,6 +133,7 @@ public WebsocketCollector websocketCollector() {
          * @return the server endpoint exporter
          */
         @Bean
+        @ConditionalOnMissingBean(ServerEndpointExporter.class)
         public ServerEndpointExporter serverEndpointExporter() {
             return new ServerEndpointExporter();
         }

diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/config/HttpSyncProperties.java b/soul-admin/src/main/java/org/dromara/soul/admin/config/HttpSyncProperties.java
@@ -0,0 +1,45 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one or more
+ *   contributor license agreements.  See the NOTICE file distributed with
+ *   this work for additional information regarding copyright ownership.
+ *   The ASF licenses this file to You under the Apache License, Version 2.0
+ *   (the "License"); you may not use this file except in compliance with
+ *   the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+package org.dromara.soul.admin.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.time.Duration;
+
+/**
+ * the http sync strategy properties.
+ * @author huangxiaofeng
+ */
+@Getter
+@Setter
+@ConfigurationProperties(prefix = "soul.sync.http")
+public class HttpSyncProperties {
+
+    /**
+     * 是否开启 http 同步策略, default: true.
+     */
+    private boolean enabled = true;
+
+    /**
+     * Periodically refresh the config data interval from the database, default: 5 minutes.
+     */
+    private Duration refreshInterval = Duration.ofMinutes(5);
+
+}
diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/config/NacosACMConfig.java b/soul-admin/src/main/java/org/dromara/soul/admin/config/NacosACMConfig.java
diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/config/NacosConfiguration.java b/soul-admin/src/main/java/org/dromara/soul/admin/config/NacosConfiguration.java
@@ -18,43 +18,44 @@
 
 package org.dromara.soul.admin.config;
 
-import java.util.Properties;
-
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Bean;
-
 import com.alibaba.nacos.api.NacosFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
 import com.alibaba.nacos.api.config.ConfigService;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+
+import java.util.Properties;
 
 /**
  * Nacos configuration.
  *
  * @author xiaoyu
  */
-@EnableConfigurationProperties(NacosConfig.class)
+@EnableConfigurationProperties(NacosProperties.class)
 public class NacosConfiguration {
 
     /**
      * register configService in spring ioc.
      *
-     * @param nacosConfig the nacos configuration
+     * @param nacosProp the nacos configuration
      * @return ConfigService {@linkplain ConfigService}
      * @throws Exception the exception
      */
     @Bean
-    public ConfigService nacosConfigService(final NacosConfig nacosConfig) throws Exception {
+    @ConditionalOnMissingBean(ConfigService.class)
+    public ConfigService nacosConfigService(final NacosProperties nacosProp) throws Exception {
         Properties properties = new Properties();
-        if (nacosConfig.getAcm() != null && nacosConfig.getAcm().isEnabled()) {
+        if (nacosProp.getAcm() != null && nacosProp.getAcm().isEnabled()) {
             //使用阿里云ACM服务
-            properties.put(PropertyKeyConst.ENDPOINT, nacosConfig.getAcm().getEndpoint());
-            properties.put(PropertyKeyConst.NAMESPACE, nacosConfig.getAcm().getNamespace());
+            properties.put(PropertyKeyConst.ENDPOINT, nacosProp.getAcm().getEndpoint());
+            properties.put(PropertyKeyConst.NAMESPACE, nacosProp.getAcm().getNamespace());
             //使用子账户ACM管理权限
-            properties.put(PropertyKeyConst.ACCESS_KEY, nacosConfig.getAcm().getAccessKey());
-            properties.put(PropertyKeyConst.SECRET_KEY, nacosConfig.getAcm().getSecretKey());
+            properties.put(PropertyKeyConst.ACCESS_KEY, nacosProp.getAcm().getAccessKey());
+            properties.put(PropertyKeyConst.SECRET_KEY, nacosProp.getAcm().getSecretKey());
         } else {
-            properties.put(PropertyKeyConst.SERVER_ADDR, nacosConfig.getUrl());
-            properties.put(PropertyKeyConst.NAMESPACE, nacosConfig.getNamespace());
+            properties.put(PropertyKeyConst.SERVER_ADDR, nacosProp.getUrl());
+            properties.put(PropertyKeyConst.NAMESPACE, nacosProp.getNamespace());
         }
         return NacosFactory.createConfigService(properties);
     }

diff --git a/...romara/soul/admin/config/NacosConfig.java → ...ra/soul/admin/config/NacosProperties.java b/...romara/soul/admin/config/NacosConfig.java → ...ra/soul/admin/config/NacosProperties.java
@@ -18,20 +18,38 @@
 
 package org.dromara.soul.admin.config;
 
-import org.springframework.boot.context.properties.ConfigurationProperties;
-
 import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
 
 /**
  * The type Nacos config.
  */
 @Data
 @ConfigurationProperties(prefix = "soul.sync.nacos")
-public class NacosConfig {
+public class NacosProperties {
 
     private String url;
 
     private String namespace;
 
-    private NacosACMConfig acm;
+    private NacosACMProperties acm;
+
+    @Getter
+    @Setter
+    public static class NacosACMProperties {
+
+        private boolean enabled;
+
+        private String endpoint;
+
+        private String namespace;
+
+        private String accessKey;
+
+        private String secretKey;
+
+    }
+
 }
diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/config/ZookeeperConfiguration.java b/soul-admin/src/main/java/org/dromara/soul/admin/config/ZookeeperConfiguration.java
@@ -19,6 +19,7 @@
 package org.dromara.soul.admin.config;
 
 import org.I0Itec.zkclient.ZkClient;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 
@@ -27,17 +28,18 @@
  *
  * @author xiaoyu(Myth)
  */
-@EnableConfigurationProperties(ZookeeperConfig.class)
+@EnableConfigurationProperties(ZookeeperProperties.class)
 public class ZookeeperConfiguration {
 
     /**
      * register zkClient in spring ioc.
      *
-     * @param zookeeperConfig the zookeeper configuration
+     * @param zookeeperProp the zookeeper configuration
      * @return ZkClient {@linkplain ZkClient}
      */
     @Bean
-    public ZkClient zkClient(final ZookeeperConfig zookeeperConfig) {
-        return new ZkClient(zookeeperConfig.getUrl(), zookeeperConfig.getSessionTimeout(), zookeeperConfig.getConnectionTimeout());
+    @ConditionalOnMissingBean(ZkClient.class)
+    public ZkClient zkClient(final ZookeeperProperties zookeeperProp) {
+        return new ZkClient(zookeeperProp.getUrl(), zookeeperProp.getSessionTimeout(), zookeeperProp.getConnectionTimeout());
     }
 }
diff --git a/...ra/soul/admin/config/ZookeeperConfig.java → ...oul/admin/config/ZookeeperProperties.java b/...ra/soul/admin/config/ZookeeperConfig.java → ...oul/admin/config/ZookeeperProperties.java
@@ -28,7 +28,7 @@
  */
 @Data
 @ConfigurationProperties(prefix = "soul.sync.zookeeper")
-public class ZookeeperConfig {
+public class ZookeeperProperties {
 
     private String url;
 

diff --git a/soul-admin/src/main/java/org/dromara/soul/admin/controller/ConfigController.java b/soul-admin/src/main/java/org/dromara/soul/admin/controller/ConfigController.java
@@ -58,9 +58,9 @@ public class ConfigController {
      */
     @GetMapping("/fetch")
     public SoulAdminResult fetchConfigs(@NotNull final String[] groupKeys) {
-        Map<String, ConfigData> result = Maps.newConcurrentMap();
+        Map<String, ConfigData<?>> result = Maps.newHashMap();
         for (String groupKey : groupKeys) {
-            ConfigData data = longPollingListener.fetchConfig(ConfigGroupEnum.valueOf(groupKey));
+            ConfigData<?> data = longPollingListener.fetchConfig(ConfigGroupEnum.valueOf(groupKey));
             result.put(groupKey, data);
         }
         return SoulAdminResult.success("success", result);