Skip to content

Commit f6f223a

Browse files
resdevdresdevd
committed
Apache Airflow version 2.1.4
1 parent 5362def commit f6f223a

File tree

6 files changed

+231
-1
lines changed

6 files changed

+231
-1
lines changed

‎.env‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
2+
3+
# CORE
4+
AIRFLOW__CORE__EXECUTOR=CeleryExecutor
5+
AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
6+
AIRFLOW__CORE__FERNET_KEY=VYCLCTGPY5xhmKY-ReNL_RPy5F6bA7ExUO_tyScpsIo=
7+
AIRFLOW__CORE__LOAD_EXAMPLES=True
8+
9+
# WEBSERVER
10+
AIRFLOW__WEBSERVER__RBAC=True
11+
AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
12+
13+
14+
# CELERY
15+
AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
16+
AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
17+
18+
19+
20+
21+
22+

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.idea/
2+

‎Dockerfile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
FROM apache/airflow:2.1.4-python3.9
2+
3+
# Install any linux packages here (Optional)
4+
5+
USER root
6+
7+
RUN apt-get update -yqq \
8+
&& apt-get install -y vim
9+
10+
11+
# Add any python libraries (Optional)
12+
#https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html?highlight=snowflake
13+
14+
USER airflow
15+
16+
RUN pip install \
17+
'python-dotenv' \
18+
'apache-airflow-providers-http==2.0.1' \
19+
'apache-airflow-providers-postgres==2.2.0'
20+
21+
22+
USER airflow

‎README.md‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,45 @@
1-
# docker-compose-airflow
1+
# docker-compose-airflow
2+
3+
_Note: Updated to Airflow version 2.1.4_
4+
5+
Docker Compose Apache Airflow (Official Docker Images) with CeleryExecutor, InitDB and InitUser
6+
7+
Ideal for local development or small scale personal deployments.
8+
9+
### Features
10+
- Add any linux packages and/or python libraries to the image.
11+
- Easy deployment in minutes.
12+
13+
14+
15+
16+
### How to deploy:
17+
18+
**Prerequisite**: Docker and docker-compose to be installed.
19+
20+
**Step 1:** Clone this Repo and open terminal.
21+
22+
**Step 2:** Go through .env file, init_airflow_setup.sh, docker-compose.yml file to change settings according to your preference. Or you can just keep them as it is for local development.
23+
24+
**Step 3:** Run `docker-compose up -d`
25+
26+
**Step 4:** Run `sh init_airflow_setup.sh` (Run this only for initial deployment)
27+
28+
**Step 5:** Go to http://localhost:8080 and login with user: _airflow_test_user_ and password: _airflow_test_password_ as specified in init_airflow_setup.sh script
29+
30+
**Step 6:** Run few dags and monitor Celery workers at http://localhost:5555
31+
32+
# Rebuild image and redeploy with postgres intact
33+
34+
docker build . -t resdev-airflow:latest
35+
36+
update image
37+
-----------
38+
cd ..
39+
docker-compose up -d --no-deps resdev-airflow
40+
41+
docker network prune
42+
43+
docker volume prune
44+
45+
docker system prune -f

‎docker-compose.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
version: '3'
2+
services:
3+
postgres:
4+
image: postgres:14.0
5+
container_name: postgres_cont
6+
environment:
7+
- POSTGRES_USER=postgres
8+
- POSTGRES_PASSWORD=postgres
9+
- POSTGRES_DB=airflow
10+
- POSTGRES_PORT=5432
11+
volumes:
12+
- database-data:/var/lib/postgresql/data/
13+
ports:
14+
- 5432:5432
15+
16+
redis:
17+
image: redis:latest
18+
container_name: redis_cont
19+
restart: always
20+
ports:
21+
- 6379:6379
22+
23+
airflow:
24+
build: .
25+
image: resdev-airflow:latest
26+
container_name: airflow_cont
27+
env_file:
28+
- .env
29+
restart: always
30+
command: webserver
31+
depends_on:
32+
- postgres
33+
ports:
34+
- 8080:8080
35+
volumes:
36+
- ./airflow-data/dags:/opt/airflow/dags
37+
- ./airflow-data/plugins:/opt/airflow/plugins
38+
- logs-volume:/opt/airflow/logs
39+
40+
airflow-scheduler:
41+
build: .
42+
image: resdev-airflow:latest
43+
container_name: airflow_scheduler_cont
44+
env_file:
45+
- .env
46+
restart: always
47+
command: scheduler
48+
depends_on:
49+
- postgres
50+
volumes:
51+
- ./airflow-data/dags:/opt/airflow/dags
52+
- ./airflow-data/plugins:/opt/airflow/plugins
53+
- logs-volume:/opt/airflow/logs
54+
55+
airflow-worker1:
56+
build: .
57+
image: resdev-airflow:latest
58+
container_name: airflow_worker1_cont
59+
env_file:
60+
- .env
61+
restart: always
62+
command: celery worker
63+
depends_on:
64+
- postgres
65+
volumes:
66+
- ./airflow-data/dags:/opt/airflow/dags
67+
- ./airflow-data/plugins:/opt/airflow/plugins
68+
- logs-volume:/opt/airflow/logs
69+
70+
airflow-worker2:
71+
build: .
72+
image: resdev-airflow:latest
73+
container_name: airflow_worker2_cont
74+
env_file:
75+
- .env
76+
restart: always
77+
command: celery worker
78+
depends_on:
79+
- postgres
80+
volumes:
81+
- ./airflow-data/dags:/opt/airflow/dags
82+
- ./airflow-data/plugins:/opt/airflow/plugins
83+
- logs-volume:/opt/airflow/logs
84+
85+
airflow-worker3:
86+
build: .
87+
image: resdev-airflow:latest
88+
container_name: airflow_worker3_cont
89+
env_file:
90+
- .env
91+
restart: always
92+
depends_on:
93+
- postgres
94+
command: celery worker
95+
volumes:
96+
- ./airflow-data/dags:/opt/airflow/dags
97+
- ./airflow-data/plugins:/opt/airflow/plugins
98+
- logs-volume:/opt/airflow/logs
99+
100+
airflow-flower:
101+
build: .
102+
image: resdev-airflow:latest
103+
container_name: airflow_flower_cont
104+
depends_on:
105+
- postgres
106+
volumes:
107+
- ./airflow-data/dags:/opt/airflow/dags
108+
- ./airflow-data/plugins:/opt/airflow/plugins
109+
- logs-volume:/opt/airflow/logs
110+
env_file:
111+
- .env
112+
restart: always
113+
command: celery flower
114+
ports:
115+
- 5555:5555
116+
117+
118+
volumes:
119+
database-data:
120+
logs-volume:

‎init_airflow_setup.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/sh
2+
3+
# WARNING: Run this script only during initial airflow db setup.
4+
5+
IS_INITDB=True
6+
AIRFLOW_USER=airflow_test_user
7+
AIRFLOW_PASSWORD=airflow_test_password
8+
[email protected]
9+
10+
if [ $IS_INITDB ]; then
11+
12+
echo "Initializing Airflow DB setup and Admin user setup because value of IS_INITDB is $IS_INITDB"
13+
echo " Airflow admin username will be $AIRFLOW_USER"
14+
15+
docker exec -ti airflow_cont airflow db init && echo "Initialized airflow DB"
16+
docker exec -ti airflow_cont airflow users create --role Admin --username $AIRFLOW_USER --password $AIRFLOW_PASSWORD -e $AIRFLOW_USER_EMAIL -f airflow -l airflow && echo "Created airflow Initial admin user with username $AIRFLOW_USER"
17+
18+
else
19+
echo "Skipping InitDB and InitUser setup because value of IS_INITDB is $IS_INITDB"
20+
fi

0 commit comments

Comments
 (0)