From bb40aaa5a66a26f94e0682e1910139a8f9cae3c5 Mon Sep 17 00:00:00 2001 From: choiseyeon Date: Mon, 3 Feb 2025 23:57:22 +0900 Subject: [PATCH 1/6] =?UTF-8?q?feat:=20=EB=8F=84=EC=BB=A4=20=EC=BB=B4?= =?UTF-8?q?=ED=8F=AC=EC=A6=88=20=ED=8C=8C=EC=9D=BC=20=EC=84=B8=ED=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 7 ++ docker-compose.yml | 218 +++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 6 ++ 3 files changed, 231 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3d0bcce --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM apache/airflow:2.8.1-python3.11 + +# USER root +COPY requirements.txt ./requirements.txt + +# USER airflow +RUN pip install -r ./requirements.txt \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..df18657 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,218 @@ +x-airflow-common: + &airflow-common + # In order to add custom dependencies or upgrade provider packages you can use your extended image. + # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml + # and uncomment the "build" line below, Then run `docker-compose build` to build the images. + image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.8.1} + # build: . + environment: + &airflow-common-env + AIRFLOW__CORE__EXECUTOR: CeleryExecutor + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow + AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 + AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' + # yamllint disable rule:line-length + # Use simple http server on scheduler for health checks + # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server + # yamllint enable rule:line-length + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks + # for other purpose (development, test and especially production usage) build/extend Airflow image. + _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} + volumes: + - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins + user: "${AIRFLOW_UID:-50000}:0" + depends_on: + &airflow-common-depends-on + redis: + condition: service_healthy + postgres: + condition: service_healthy + +services: + postgres: + image: postgres:15 + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + ports: + - 5432:5432 + volumes: + - postgres-db-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "airflow"] + interval: 10s + retries: 5 + start_period: 5s + restart: always + + redis: + image: redis:latest + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + + airflow-webserver: + <<: *airflow-common + command: webserver + build: + dockerfile: Dockerfile + ports: + - "8080:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-scheduler: + <<: *airflow-common + command: scheduler + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-worker: + <<: *airflow-common + command: celery worker + healthcheck: + # yamllint disable rule:line-length + test: + - "CMD-SHELL" + - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + environment: + <<: *airflow-common-env + # Required to handle warm shutdown of the celery workers properly + # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-triggerer: + <<: *airflow-common + command: triggerer + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-init: + <<: *airflow-common + entrypoint: /bin/bash + # yamllint disable rule:line-length + command: + - -c + - | + if [[ -z "${AIRFLOW_UID}" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" + echo "If you are on Linux, you SHOULD follow the instructions below to set " + echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." + echo "For other operating systems you can get rid of the warning with manually created .env file:" + echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" + echo + fi + one_meg=1048576 + mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) + cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) + disk_available=$$(df / | tail -1 | awk '{print $$4}') + warning_resources="false" + if (( mem_available < 4000 )) ; then + echo + echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" + echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" + echo + warning_resources="true" + fi + if (( cpus_available < 2 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" + echo "At least 2 CPUs recommended. You have $${cpus_available}" + echo + warning_resources="true" + fi + if (( disk_available < one_meg * 10 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" + echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" + echo + warning_resources="true" + fi + if [[ $${warning_resources} == "true" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" + echo "Please follow the instructions to increase amount of resources available:" + echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" + echo + fi + mkdir -p /sources/logs /sources/dags /sources/plugins + chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + exec /entrypoint airflow version + # yamllint enable rule:line-length + environment: + <<: *airflow-common-env + _AIRFLOW_DB_MIGRATE: 'true' + _AIRFLOW_WWW_USER_CREATE: 'true' + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} + user: "0:0" + volumes: + - ${AIRFLOW_PROJ_DIR:-.}:/sources + + airflow-cli: + <<: *airflow-common + profiles: + - debug + environment: + <<: *airflow-common-env + CONNECTION_CHECK_MAX_COUNT: "0" + # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 + command: + - bash + - -c + - airflow + +volumes: + postgres-db-volume: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b0bac77 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +pylint +aiohttp +requests +apache-airflow +apache-airflow-providers-mysql +apache-airflow-providers-postgres \ No newline at end of file From b2c2544365e5a56e0ca478bf15a1b681178cb1d0 Mon Sep 17 00:00:00 2001 From: choiseyeon Date: Tue, 4 Feb 2025 21:38:18 +0900 Subject: [PATCH 2/6] =?UTF-8?q?feat:=20=ED=94=84=EB=A1=9C=EC=A0=9D?= =?UTF-8?q?=ED=8A=B8=20=EA=B5=AC=EC=A1=B0=20=EC=84=B8=ED=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +++ {test => api-test}/binance_1m.py | 0 {test => api-test}/upbit_1m.py | 0 dags/collectors/binance_collector.py | 0 dags/collectors/utils/utils.py | 0 dags/configs/airflow.cfg | 0 dags/configs/config.yaml | 0 dags/loaders/parquet_loader.py | 0 dags/processors/transform.py | 0 dags/tests/test_collectors.py | 0 10 files changed, 3 insertions(+) create mode 100644 .gitignore rename {test => api-test}/binance_1m.py (100%) rename {test => api-test}/upbit_1m.py (100%) create mode 100644 dags/collectors/binance_collector.py create mode 100644 dags/collectors/utils/utils.py create mode 100644 dags/configs/airflow.cfg create mode 100644 dags/configs/config.yaml create mode 100644 dags/loaders/parquet_loader.py create mode 100644 dags/processors/transform.py create mode 100644 dags/tests/test_collectors.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f859bb4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +config +logs +plugins \ No newline at end of file diff --git a/test/binance_1m.py b/api-test/binance_1m.py similarity index 100% rename from test/binance_1m.py rename to api-test/binance_1m.py diff --git a/test/upbit_1m.py b/api-test/upbit_1m.py similarity index 100% rename from test/upbit_1m.py rename to api-test/upbit_1m.py diff --git a/dags/collectors/binance_collector.py b/dags/collectors/binance_collector.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/collectors/utils/utils.py b/dags/collectors/utils/utils.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/configs/airflow.cfg b/dags/configs/airflow.cfg new file mode 100644 index 0000000..e69de29 diff --git a/dags/configs/config.yaml b/dags/configs/config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/dags/loaders/parquet_loader.py b/dags/loaders/parquet_loader.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/processors/transform.py b/dags/processors/transform.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/tests/test_collectors.py b/dags/tests/test_collectors.py new file mode 100644 index 0000000..e69de29 From 3d9575e6374b8a42706a31a19e9b864a7be728c4 Mon Sep 17 00:00:00 2001 From: choiseyeon Date: Tue, 4 Feb 2025 23:48:53 +0900 Subject: [PATCH 3/6] =?UTF-8?q?feat:=20.env=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + dags/configs/airflow.cfg | 0 dags/configs/config.yaml | 0 docker-compose.yml | 128 +++++++++------------------------------ requirements.txt | 9 +-- 5 files changed, 35 insertions(+), 104 deletions(-) delete mode 100644 dags/configs/airflow.cfg delete mode 100644 dags/configs/config.yaml diff --git a/.gitignore b/.gitignore index f859bb4..c25b7cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.env +dags/__pycache__ config logs plugins \ No newline at end of file diff --git a/dags/configs/airflow.cfg b/dags/configs/airflow.cfg deleted file mode 100644 index e69de29..0000000 diff --git a/dags/configs/config.yaml b/dags/configs/config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/docker-compose.yml b/docker-compose.yml index df18657..83e6302 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,24 @@ x-airflow-common: &airflow-common - # In order to add custom dependencies or upgrade provider packages you can use your extended image. - # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml - # and uncomment the "build" line below, Then run `docker-compose build` to build the images. - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.8.1} - # build: . + image: ${AIRFLOW_IMAGE_NAME} environment: &airflow-common-env - AIRFLOW__CORE__EXECUTOR: CeleryExecutor - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow - AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: '' - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: 'false' - AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' - # yamllint disable rule:line-length - # Use simple http server on scheduler for health checks - # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server - # yamllint enable rule:line-length - AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' - # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks - # for other purpose (development, test and especially production usage) build/extend Airflow image. - _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} + AIRFLOW__CORE__EXECUTOR: ${AIRFLOW__CORE__EXECUTOR} + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN} + AIRFLOW__CELERY__RESULT_BACKEND: ${AIRFLOW__CELERY__RESULT_BACKEND} + AIRFLOW__CELERY__BROKER_URL: ${AIRFLOW__CELERY__BROKER_URL} + AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: ${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION} + AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES} + AIRFLOW__API__AUTH_BACKENDS: ${AIRFLOW__API__AUTH_BACKENDS} + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: ${AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK} + _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS} volumes: - - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins - user: "${AIRFLOW_UID:-50000}:0" + - ${AIRFLOW_PROJ_DIR}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR}/plugins:/opt/airflow/plugins + user: "${AIRFLOW_UID}:0" depends_on: &airflow-common-depends-on redis: @@ -40,15 +30,15 @@ services: postgres: image: postgres:15 environment: - POSTGRES_USER: airflow - POSTGRES_PASSWORD: airflow - POSTGRES_DB: airflow + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} ports: - 5432:5432 volumes: - postgres-db-volume:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "airflow"] + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] interval: 10s retries: 5 start_period: 5s @@ -65,7 +55,7 @@ services: retries: 50 start_period: 30s restart: always - + airflow-webserver: <<: *airflow-common command: webserver @@ -88,12 +78,6 @@ services: airflow-scheduler: <<: *airflow-common command: scheduler - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on @@ -103,19 +87,8 @@ services: airflow-worker: <<: *airflow-common command: celery worker - healthcheck: - # yamllint disable rule:line-length - test: - - "CMD-SHELL" - - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s environment: <<: *airflow-common-env - # Required to handle warm shutdown of the celery workers properly - # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation DUMB_INIT_SETSID: "0" restart: always depends_on: @@ -126,12 +99,6 @@ services: airflow-triggerer: <<: *airflow-common command: triggerer - healthcheck: - test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on @@ -141,65 +108,27 @@ services: airflow-init: <<: *airflow-common entrypoint: /bin/bash - # yamllint disable rule:line-length command: - -c - | if [[ -z "${AIRFLOW_UID}" ]]; then echo echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" - echo "If you are on Linux, you SHOULD follow the instructions below to set " - echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." - echo "For other operating systems you can get rid of the warning with manually created .env file:" - echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" - echo - fi - one_meg=1048576 - mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) - cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) - disk_available=$$(df / | tail -1 | awk '{print $$4}') - warning_resources="false" - if (( mem_available < 4000 )) ; then - echo - echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" - echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" - echo - warning_resources="true" - fi - if (( cpus_available < 2 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" - echo "At least 2 CPUs recommended. You have $${cpus_available}" - echo - warning_resources="true" - fi - if (( disk_available < one_meg * 10 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" - echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" - echo - warning_resources="true" - fi - if [[ $${warning_resources} == "true" ]]; then - echo - echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" - echo "Please follow the instructions to increase amount of resources available:" - echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" + echo "See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" echo fi mkdir -p /sources/logs /sources/dags /sources/plugins chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} exec /entrypoint airflow version - # yamllint enable rule:line-length environment: <<: *airflow-common-env - _AIRFLOW_DB_MIGRATE: 'true' - _AIRFLOW_WWW_USER_CREATE: 'true' - _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} - _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} + _AIRFLOW_DB_MIGRATE: ${_AIRFLOW_DB_MIGRATE} + _AIRFLOW_WWW_USER_CREATE: ${_AIRFLOW_WWW_USER_CREATE} + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} user: "0:0" volumes: - - ${AIRFLOW_PROJ_DIR:-.}:/sources + - ${AIRFLOW_PROJ_DIR}:/sources airflow-cli: <<: *airflow-common @@ -208,11 +137,10 @@ services: environment: <<: *airflow-common-env CONNECTION_CHECK_MAX_COUNT: "0" - # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 command: - bash - -c - airflow volumes: - postgres-db-volume: \ No newline at end of file + postgres-db-volume: diff --git a/requirements.txt b/requirements.txt index b0bac77..84003ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -pylint -aiohttp -requests apache-airflow apache-airflow-providers-mysql -apache-airflow-providers-postgres \ No newline at end of file +apache-airflow-providers-postgres + +pylint +aiohttp +requests \ No newline at end of file From cde669af74cf0a77d8bbf16dcfc400685c663c64 Mon Sep 17 00:00:00 2001 From: choiseyeon Date: Wed, 5 Feb 2025 01:24:07 +0900 Subject: [PATCH 4/6] =?UTF-8?q?feat:=20pylint=20=EC=84=B8=ED=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/dag.py | 0 dags/settings.json | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 dags/dag.py create mode 100644 dags/settings.json diff --git a/dags/dag.py b/dags/dag.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/settings.json b/dags/settings.json new file mode 100644 index 0000000..cb0e373 --- /dev/null +++ b/dags/settings.json @@ -0,0 +1,8 @@ +{ + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.linting.pylintArgs": [ + "--disable=C0114,C0115,C0116", + "--max-line-length=120" + ] +} \ No newline at end of file From 76467343d748ce26c078e7fc557a3666bc9d649b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=B5=9C=EC=84=B8=EC=97=B0=20=28Lena=29?= Date: Wed, 5 Feb 2025 02:44:29 +0900 Subject: [PATCH 5/6] =?UTF-8?q?hotfix(feat):=20=EC=9D=98=EC=A1=B4=EC=84=B1?= =?UTF-8?q?=20=EC=84=A4=EC=B9=98=EB=A5=BC=20=EC=B6=94=EA=B0=80=ED=95=A9?= =?UTF-8?q?=EB=8B=88=EB=8B=A4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ".github/workflows/\bpr.yml" | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git "a/.github/workflows/\bpr.yml" "b/.github/workflows/\bpr.yml" index 9167e64..d704146 100644 --- "a/.github/workflows/\bpr.yml" +++ "b/.github/workflows/\bpr.yml" @@ -15,7 +15,9 @@ jobs: python-version: '3.11' - name: Install dependencies - run: pip install pylint + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run Pylint run: pylint . From 625f131fc93a3384e336f3af24ececca516a55ee Mon Sep 17 00:00:00 2001 From: choiseyeon Date: Wed, 5 Feb 2025 02:47:01 +0900 Subject: [PATCH 6/6] =?UTF-8?q?feat:=20=EB=B6=88=ED=95=84=EC=9A=94?= =?UTF-8?q?=ED=95=9C=20=EC=BD=94=EB=93=9C=EB=A5=BC=20=EC=A0=9C=EA=B1=B0?= =?UTF-8?q?=ED=95=A9=EB=8B=88=EB=8B=A4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 -- dags/settings.json | 8 -------- 2 files changed, 10 deletions(-) delete mode 100644 dags/settings.json diff --git a/Dockerfile b/Dockerfile index 3d0bcce..2a54b3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,5 @@ FROM apache/airflow:2.8.1-python3.11 -# USER root COPY requirements.txt ./requirements.txt -# USER airflow RUN pip install -r ./requirements.txt \ No newline at end of file diff --git a/dags/settings.json b/dags/settings.json deleted file mode 100644 index cb0e373..0000000 --- a/dags/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.linting.pylintArgs": [ - "--disable=C0114,C0115,C0116", - "--max-line-length=120" - ] -} \ No newline at end of file