diff --git "a/.github/workflows/\bpr.yml" "b/.github/workflows/\bpr.yml" index 9167e64..d704146 100644 --- "a/.github/workflows/\bpr.yml" +++ "b/.github/workflows/\bpr.yml" @@ -15,7 +15,9 @@ jobs: python-version: '3.11' - name: Install dependencies - run: pip install pylint + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run Pylint run: pylint . diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c25b7cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.env +dags/__pycache__ +config +logs +plugins \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2a54b3e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM apache/airflow:2.8.1-python3.11 + +COPY requirements.txt ./requirements.txt + +RUN pip install -r ./requirements.txt \ No newline at end of file diff --git a/test/binance_1m.py b/api-test/binance_1m.py similarity index 100% rename from test/binance_1m.py rename to api-test/binance_1m.py diff --git a/test/upbit_1m.py b/api-test/upbit_1m.py similarity index 100% rename from test/upbit_1m.py rename to api-test/upbit_1m.py diff --git a/dags/collectors/binance_collector.py b/dags/collectors/binance_collector.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/collectors/utils/utils.py b/dags/collectors/utils/utils.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/dag.py b/dags/dag.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/loaders/parquet_loader.py b/dags/loaders/parquet_loader.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/processors/transform.py b/dags/processors/transform.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/tests/test_collectors.py b/dags/tests/test_collectors.py new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..83e6302 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,146 @@ +x-airflow-common: + &airflow-common + image: ${AIRFLOW_IMAGE_NAME} + environment: + &airflow-common-env + AIRFLOW__CORE__EXECUTOR: ${AIRFLOW__CORE__EXECUTOR} + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN} + AIRFLOW__CELERY__RESULT_BACKEND: ${AIRFLOW__CELERY__RESULT_BACKEND} + AIRFLOW__CELERY__BROKER_URL: ${AIRFLOW__CELERY__BROKER_URL} + AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: ${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION} + AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES} + AIRFLOW__API__AUTH_BACKENDS: ${AIRFLOW__API__AUTH_BACKENDS} + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: ${AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK} + _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS} + volumes: + - ${AIRFLOW_PROJ_DIR}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR}/plugins:/opt/airflow/plugins + user: "${AIRFLOW_UID}:0" + depends_on: + &airflow-common-depends-on + redis: + condition: service_healthy + postgres: + condition: service_healthy + +services: + postgres: + image: postgres:15 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - 5432:5432 + volumes: + - postgres-db-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] + interval: 10s + retries: 5 + start_period: 5s + restart: always + + redis: + image: redis:latest + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + + airflow-webserver: + <<: *airflow-common + command: webserver + build: + dockerfile: Dockerfile + ports: + - "8080:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-scheduler: + <<: *airflow-common + command: scheduler + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-worker: + <<: *airflow-common + command: celery worker + environment: + <<: *airflow-common-env + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-triggerer: + <<: *airflow-common + command: triggerer + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-init: + <<: *airflow-common + entrypoint: /bin/bash + command: + - -c + - | + if [[ -z "${AIRFLOW_UID}" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" + echo "See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" + echo + fi + mkdir -p /sources/logs /sources/dags /sources/plugins + chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + exec /entrypoint airflow version + environment: + <<: *airflow-common-env + _AIRFLOW_DB_MIGRATE: ${_AIRFLOW_DB_MIGRATE} + _AIRFLOW_WWW_USER_CREATE: ${_AIRFLOW_WWW_USER_CREATE} + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} + user: "0:0" + volumes: + - ${AIRFLOW_PROJ_DIR}:/sources + + airflow-cli: + <<: *airflow-common + profiles: + - debug + environment: + <<: *airflow-common-env + CONNECTION_CHECK_MAX_COUNT: "0" + command: + - bash + - -c + - airflow + +volumes: + postgres-db-volume: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..84003ff --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +apache-airflow +apache-airflow-providers-mysql +apache-airflow-providers-postgres + +pylint +aiohttp +requests \ No newline at end of file