Complete Rework using TDigest & Flink's MapState

.circleci/config.yml

@@ -0,0 +1,55 @@
+# Java Maven CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-java/ for more details
+#
+# Continuous Integration!
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/openjdk:8-jdk
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      MAVEN_OPTS: -Xmx3200m
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "pom.xml" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run: mvn package -P build-jar dependency:go-offline
+
+      - save_cache:
+          paths:
+            - ~/.m2
+          key: v1-dependencies-{{ checksum "pom.xml" }}
+
+      # run tests!
+      #- run: mvn integration-test
+
+      - run: |
+          set -xu
+          mkdir -p /tmp/artifacts
+          cp target/smartgrid*.jar /tmp/artifacts
+
+      # Save artifacts
+      - store_artifacts:
+          path: /tmp/artifacts
+          destination: build
+
+
+

docker/Flink-Hbase-Kafka/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '2.1'
+services:
+  flink-kafka-hbase:
+    image: quay.io/koldbyte/flink-cluster
+    container_name: flink_to_hbase
+    hostname: hbase-docker
+    ports:
+      - 8080:8080
+      - 8085:8085
+      - 9090:9090
+      - 9092:9092
+      - 9095:9095
+      - 2181:2181
+      - 16000:16000
+      - 16010:16010
+      - 16020:16020
+      - 16030:16030
+      - 6123:6123
+      - 7203:7203
+      - 8081:8081
+      - 8090:8090
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: hbase-docker
+    volumes:
+      - /data
+    external_links:
+      - elasticsearch
+      - kibana
+      - kafka-manager

docker/docker-compose.yml

@@ -0,0 +1,67 @@
+version: '2.1'
+services:
+  flink-kafka-hbase:
+    domainname: smartgrid.com
+    extends:
+      service: flink-kafka-hbase
+      file: ./Flink-Hbase-Kafka/docker-compose.yml
+    networks:
+      - smartgrid.com
+    volumes:
+      - /app/smartgrid/data:/data
+    links:
+      - elasticsearch
+      - elasticsearch2
+
+  elasticsearch:
+    domainname: smartgrid.com
+    extends:
+      service: elasticsearch
+      file: ./elasticsearch/docker-compose.yml
+    networks:
+      - smartgrid.com
+    volumes:
+      - /app/smartgrid/es-data:/usr/share/elasticsearch/data
+
+  elasticsearch2:
+    domainname: smartgrid.com
+    extends:
+      service: elasticsearch2
+      file: ./elasticsearch/docker-compose.yml
+    networks:
+      - smartgrid.com
+    volumes:
+      - /app/smartgrid/es-data-2:/usr/share/elasticsearch/data
+
+  kibana:
+    domainname: smartgrid.com
+    extends:
+      service: kibana
+      file: ./kibana/docker-compose.yml
+    networks:
+      - smartgrid.com
+    depends_on:
+      - elasticsearch
+    links:
+      - elasticsearch
+      - elasticsearch2
+
+  kafka_manager:
+    domainname: smartgrid.com
+    extends:
+      service: kafka_manager
+      file: ./kafka-manager/docker-compose.yml
+    networks:
+      - smartgrid.com
+    volumes:
+      - /app/smartgrid/kafka-manager:/kafka-manager/configuration
+    depends_on:
+      - flink-kafka-hbase
+    links:
+      - flink-kafka-hbase
+      - "flink-kafka-hbase:hbase-docker"
+
+networks:
+  smartgrid.com:
+    driver: bridge
+    name: smartgrid.com

docker/elasticsearch/docker-compose.yml

@@ -1,54 +1,59 @@
-elasticsearch:
-  image: docker.elastic.co/elasticsearch/elasticsearch:5.1.2
-  ports:
-    - 9200:9200
-    - 9300:9300
-  container_name: elasticsearch
-  ulimits:
-    memlock:
-      soft: -1
-      hard: -1
-  mem_limit: 1g
-  environment:
-    - cluster.name=docker-cluster
-    - node.name=one
-    - bootstrap.memory_lock=false
-    - xpack.security.enabled=false
-    - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    - network.publish_host=192.168.99.100
-    - transport.publish_port=9300
-    - transport.host=0.0.0.0
-    - transport.tcp.port=9300
-    - network.host=0.0.0.0
-    - http.host=0.0.0.0
-    - http.port=9200
-  volumes:
-    - /usr/share/elasticsearch/data
+version: '2.1'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.1.2
+    hostname: elasticsearch
+    ports:
+      - 9200:9200
+      - 9300:9300
+    container_name: elasticsearch
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    environment:
+      - cluster.name=docker-cluster
+      - node.name=one
+      - bootstrap.memory_lock=false
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - network.publish_host=elasticsearch
+      - transport.publish_port=9300
+      - transport.host=0.0.0.0
+      - transport.tcp.port=9300
+      - network.host=0.0.0.0
+      - http.host=0.0.0.0
+      - http.port=9200
+    volumes:
+      - /usr/share/elasticsearch/data
 
-elasticsearch2:
-  image: docker.elastic.co/elasticsearch/elasticsearch:5.1.2
-  ports:
-    - 9201:9200
-    - 9301:9300
-  container_name: elasticsearch2
-  ulimits:
-    memlock:
-      soft: -1
-      hard: -1
-  mem_limit: 1g
-  environment:
-    - cluster.name=docker-cluster
-    - node.name=two
-    - bootstrap.memory_lock=false
-    - xpack.security.enabled=false
-    - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    - network.publish_host=192.168.99.100
-    - transport.publish_port=9301
-    - "discovery.zen.ping.unicast.hosts=192.168.99.100"
-    - "discovery.zen.minimum_master_nodes=2"
-    - network.host=0.0.0.0
-    - transport.host=0.0.0.0
-    - transport.tcp.port=9300
-    - http.host=0.0.0.0
-  volumes:
-    - /usr/share/elasticsearch/data
+  elasticsearch2:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.1.2
+    hostname: elasticsearch2
+    ports:
+      - 9201:9200
+      - 9301:9300
+    container_name: elasticsearch2
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    mem_limit: 1g
+    environment:
+      - cluster.name=docker-cluster
+      - node.name=two
+      - bootstrap.memory_lock=false
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - network.publish_host=elasticsearch2
+      - transport.publish_port=9300
+      - "discovery.zen.ping.unicast.hosts=elasticsearch"
+      - "discovery.zen.minimum_master_nodes=2"
+      - network.host=0.0.0.0
+      - transport.host=0.0.0.0
+      - transport.tcp.port=9300
+      - http.host=0.0.0.0
+      - http.port=9200
+    volumes:
+      - /usr/share/elasticsearch/data

docker/kafka-manager/Docker-compose.yml → docker/kafka-manager/docker-compose.yml

@@ -1,9 +1,12 @@
-services:
-  kafka_manager:
-    image: hlebalbau/kafka-manager
-    ports:
-      - "9000:9000"
-    environment:
-      ZK_HOSTS: "zoo:2181"
-      APPLICATION_SECRET: "random-secret"
+version: '2.1'
+services:
+  kafka_manager:
+    image: hlebalbau/kafka-manager
+    hostname: kafka-manager
+    container_name: kafka-manager
+    ports:
+      - "9000:9000"
+    environment:
+      ZK_HOSTS: "hbase-docker:2181/kafka"
+      APPLICATION_SECRET: "random-secret"
     command: -Dpidfile.path=/dev/null

docker/kibana/Docker-compose.yml → docker/kibana/docker-compose.yml

@@ -1,11 +1,14 @@
-version: '2'
-services:
-  kibana:
-    image: kibana:5.1.2
-    container_name: kibana 
-    restart: always
-    network_mode: "bridge"
-    ports:
-      - "5601:5601"
-    external_links:
-      - elasticsearch:elasticsearch
+version: '2.1'
+services:
+  kibana:
+    image: kibana:5.1.2
+    hostname: kibana
+    container_name: kibana 
+    restart: always
+    ports:
+      - "5601:5601"
+    environment:
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
+      - SERVER_NAME=elasticsearch
+    external_links:
+      - elasticsearch:elasticsearch

pom.xml

@@ -58,6 +58,7 @@ under the License.
         <!-- Custom Added -->
         <elasticsearch.version>5.1.2</elasticsearch.version>
         <kafka.version>0.11.0.2</kafka.version>
+        <tdigest.version>3.2</tdigest.version>
         <!-- /Custom Added -->
     </properties>
 
@@ -159,6 +160,11 @@ under the License.
             <artifactId>flink-connector-elasticsearch5_${scala.binary.version}</artifactId>
             <version>${flink.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.tdunning</groupId>
+            <artifactId>t-digest</artifactId>
+            <version>${tdigest.version}</version>
+        </dependency>
         <!-- /Custom Added -->
     </dependencies>