feat: Add docker-compose files

Author: 1ambda

.gitignore

@@ -0,0 +1 @@
+./idea

Makefile

@@ -0,0 +1,68 @@
+TAG = "Makefile"
+
+MYSQLCLIENT = mycli
+DOCKER_HOST_IP := $(shell ipconfig getifaddr en0)
+
+##
+## Compose
+##
+
+.PHONY: compose.prepare
+compose.prepare:
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Preparing docker-compose"
+	@ echo "-----------------------------------------\n"
+	@ echo "export DOCKER_HOST_IP=$(DOCKER_HOST_IP)"
+	@ echo "\n-----------------------------------------"
+	@ echo ""
+
+.PHONY: compose.storage
+compose.storage: compose.prepare
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Running docker-compose"
+	@ docker stop $(docker ps -a -q) || true
+	@ docker rm -f $(docker ps -a -q) || true
+	@ docker volume rm $(docker volume ls -f dangling=true -q) || true
+	@ docker compose -f docker-compose.storage.yml rm -fsv || true
+	@ DOCKER_HOST_IP=$(DOCKER_HOST_IP) docker compose \
+		-f docker-compose.storage.yml \
+		up
+
+.PHONY: compose.spark
+compose.spark: compose.prepare
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Running docker-compose"
+	@ docker stop $(docker ps -a -q) || true
+	@ docker rm -f $(docker ps -a -q) || true
+	@ docker volume rm $(docker volume ls -f dangling=true -q) || true
+	@ docker compose -f docker-compose.spark.yml rm -fsv || true
+	@ DOCKER_HOST_IP=$(DOCKER_HOST_IP) docker compose \
+		-f docker-compose.spark.yml \
+		up
+
+.PHONY: compose.clean
+compose.clean:
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Starting: Cleaning docker resources"
+	@ echo "-----------------------------------------\n"
+	@ docker stop `docker ps -a -q` || true
+	@ docker rm -f `docker ps -a -q` || true
+	@ docker rmi -f `docker images --quiet --filter "dangling=true"` || true
+	@ docker volume rm `docker volume ls -f dangling=true -q` || true
+	@ rm -rf ./docker-volumes
+	@ docker network rm `docker network ls -q` || true
+	@ echo ""
+	@ rm -rf metastore_db
+	@ echo "\n-----------------------------------------"
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Finished: Cleaning docker resources"
+
+##
+## Storage CLIs
+##
+
+.PHONY: mysql
+mysql:
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Connecting to mysql"
+	@ $(MYSQLCLIENT) -u root -h localhost ad_stat -p root
+
+.PHONY: redis
+redis:
+	@ echo "[$(TAG)] ($(shell TZ=UTC date -u '+%H:%M:%S')) - Connecting to redis"
+	@ redis-cli -a credential
+

README.md

@@ -0,0 +1,2 @@
+# Practical Data Pipeline (Code)
+

_script/docker-mysql/conf/my.cnf

@@ -0,0 +1,12 @@
+[client]
+default-character-set = utf8mb4
+
+[mysql]
+default-character-set = utf8mb4
+
+[mysqld]
+character-set-client-handshake = FALSE
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci
+default-storage-engine=InnoDB
+default-time-zone = '+09:00'

_script/docker-mysql/sql/001_create_database.sql

@@ -0,0 +1 @@
+CREATE DATABASE pipeline;

_script/docker-mysql/sql/002_create_table.sql

@@ -0,0 +1,17 @@
+CREATE TABLE `ListingMeta`
+(
+    -- primary key
+    `listing_id`      BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+    `listing_name`    VARCHAR(240) NULL,
+    `listing_desc`    TEXT NULL,
+    `listing_summary` TEXT NULL,
+    `listing_url`     TEXT NULL,
+
+    -- FK columns
+
+    -- common
+    `created_at`      datetime DEFAULT CURRENT_TIMESTAMP NOT NULL
+
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4
+  COLLATE = utf8mb4_unicode_ci;

datasets/csv/marketing_campaign.csv

@@ -0,0 +1,60 @@
+version: '3.6'
+services:
+  spark-master:
+    image: bde2020/spark-master:3.1.1-hadoop3.2
+    container_name: spark-master
+    ports:
+      - "8080:8080"
+      - "7077:7077"
+    volumes:
+      - ./_script/docker-spark/apps:/opt/spark-apps
+      - ./_script/docker-spark/data:/opt/spark-data
+    environment:
+      - INIT_DAEMON_STEP=setup_spark
+
+  spark-worker-1:
+    image: bde2020/spark-worker:3.1.1-hadoop3.2
+    container_name: spark-worker-1
+    depends_on:
+      - spark-master
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./_script/docker-spark/apps:/opt/spark-apps
+      - ./_script/docker-spark/data:/opt/spark-data
+    environment:
+      - "SPARK_MASTER=spark://spark-master:7077"
+  spark-worker-2:
+
+    image: bde2020/spark-worker:3.1.1-hadoop3.2
+    container_name: spark-worker-2
+    depends_on:
+      - spark-master
+    ports:
+      - "8082:8081"
+    volumes:
+      - ./_script/docker-spark/apps:/opt/spark-apps
+      - ./_script/docker-spark/data:/opt/spark-data
+    environment:
+      - "SPARK_MASTER=spark://spark-master:7077"
+
+  spark-history-server:
+    image: bde2020/spark-history-server:3.1.1-hadoop3.2
+    container_name: spark-history-server
+    depends_on:
+      - spark-master
+    ports:
+      - "18081:18081"
+    volumes:
+      - /tmp/spark-events-local:/tmp/spark-events
+      - ./_script/docker-spark/apps:/opt/spark-apps
+      - ./_script/docker-spark/data:/opt/spark-data
+  storage-postgres:
+    image: postgres:11.7-alpine
+    container_name: storage-postgers
+    depends_on:
+      - spark-master
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_PASSWORD=root

docker-compose.spark.yml

@@ -0,0 +1,25 @@
+version: '3.6'
+services:
+  mysql:
+    image: mysql:8
+    container_name: mysql
+    restart: always
+    ports:
+      - 3306:3306
+    environment:
+      - MYSQL_DATABASE=pipeline
+      - MYSQL_ROOT_PASSWORD=root
+      - LANG=C.UTF-8
+    volumes:
+      - ./_script/docker-mysql/conf/:/etc/mysql/conf.d
+      - ./_script/docker-mysql/sql/:/docker-entrypoint-initdb.d
+    command: --sql_mode=''
+
+  redis:
+    image: redis:5
+    container_name: redis
+    restart: always
+    command: redis-server # --requirepass credential
+    ports:
+      - 6379:6379
+