Add pre-commit hooks (#19)

This PR adds pre-commit hooks and adds them to CI

---------

Co-authored-by: Kaxil Naik <[email protected]>

.circleci/config.yml

@@ -1,5 +1,3 @@
-# Use the latest 2.1 version of CircleCI pipeline process engine.
-# See: https://circleci.com/docs/configuration-reference
 version: 2.1
 
 all_branches_and_version_tag: &all_branches_and_version_tag
@@ -36,12 +34,25 @@ jobs:
           name: Run tests
           command: cd api && poetry run pytest ../tests
 
+  precommit:
+    executor: docker-executor
+    steps:
+      - checkout
+      - run:
+          name: Install Pre-commit
+          command: pip install pre-commit
+      - run:
+          name: Run pre-commit hooks
+          command: pre-commit run --all-files
+
 workflows:
-    tests:
-      jobs:
+  tests:
+    jobs:
       - test:
           name: test-python<< matrix.python_version >>
           matrix:
             parameters:
               python_version: [ "3.11" ]
           <<: *all_branches_and_version_tag
+      - precommit:
+          <<: *all_branches_and_version_tag

.gitignore

@@ -1,2 +1,49 @@
+.git
+coverage.xml
+**/.coverage*
+**/.python-version
+test-connections.yaml
+.idea
+.env
+airflow_settings.yaml
+*.pyc
+*/__pycache__/*
+.idea%
+/build/
+/dist/
+**/dist/
+venv
+env
+dumps
+.vscode
+.nox
+
+# Benchmark
+airflow.db
+airflow.cfg
+unittests.db
+unittests.cfg
+
+# VIM
+*.sw[a-z]
+
+# MacOS
 .DS_Store
-**pycache**
+*/.DS_Store
+
+# docker
+**/dev/dags
+**/dev/logs
+**/dev/plugins
+
+# Airflow
+**/.airflowignore
+
+*.egg-info/
+
+# sphinx-related
+/docs/_build/
+/docs/_build/*
+/docs/autoapi/
+/.nox/
+.idea

.pre-commit-config.yaml

@@ -0,0 +1,83 @@
+---
+ci:
+  skip:
+    - mypy  # requires additional dependencies in ci
+    - identity  # output is too verbose for ci; pre-commit.ci truncates almost all output after that
+default_stages: [commit, push]
+default_language_version:
+  python: python3  # force all unspecified python hooks to run python3
+minimum_pre_commit_version: "1.20.0"
+exclude: ^.*poetry.lock|ui/package-lock.json$
+repos:
+  - repo: meta
+    hooks:
+      - id: identity
+      - id: check-hooks-apply
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: debug-statements
+        types: [python]
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-no-log-warn
+        types: [python]
+      - id: python-check-mock-methods
+        types: [python]
+
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.1
+    hooks:
+      - id: forbid-crlf
+      - id: remove-crlf
+      - id: forbid-tabs
+        exclude: ^mk/|^docs/Makefile|^Makefile$
+      - id: remove-tabs
+        exclude: ^mk/|^docs/Makefile|^Makefile$
+
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+        args: ["--config", "pyproject.toml"]
+        types: [python]
+
+  - repo: https://github.com/asottile/blacken-docs
+    rev: 1.15.0
+    hooks:
+      - id: blacken-docs
+        alias: black
+        additional_dependencies: [black>=22.10.0]
+        types: [python]
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.278'
+    hooks:
+      - id: ruff
+        args:
+          - --config=./ruff.toml
+        types: [python]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.5
+    hooks:
+      - id: codespell
+        name: Run codespell to check for common misspellings in files
+        language: python
+        types: [text]
+        exclude: ^mk/.*\.mk$|^tests/modified_constraint_file.txt$
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.9.0
+    hooks:
+      - id: pyupgrade
+        args: [--py39-plus]
+        types: [python]

airflow/.astro/test_dag_integrity_default.py

@@ -1,91 +1,94 @@
 """Test the validity of all DAGs. **USED BY DEV PARSE COMMAND DO NOT EDIT**"""
-from contextlib import contextmanager
 import logging
 import os
+from contextlib import contextmanager
 
 import pytest
 
-from airflow.models import DagBag, Variable, Connection
 from airflow.hooks.base import BaseHook
-
+from airflow.models import Connection, DagBag, Variable
 
 # The following code patches errors caused by missing OS Variables, Airflow Connections, and Airflow Variables
 
+
 # =========== MONKEYPATCH BaseHook.get_connection() ===========
-def basehook_get_connection_monkeypatch(key: str,*args, **kwargs):
+def basehook_get_connection_monkeypatch(key: str, *args, **kwargs):
     print(f"Attempted to fetch connection during parse returning an empty Connection object for {key}")
     return Connection(key)
 
 
 BaseHook.get_connection = basehook_get_connection_monkeypatch
 # # =========== /MONKEYPATCH BASEHOOK.GET_CONNECTION() ===========
 
+
 # =========== MONKEYPATCH OS.GETENV() ===========
 def os_getenv_monkeypatch(key: str, *args, default=None, **kwargs):
-	print(f"Attempted to fetch os environment variable during parse, returning a mocked value for {key}")
-	if key == 'JENKINS_HOME' and default is None: # fix https://github.com/astronomer/astro-cli/issues/601
-		return None
-	if default:
-		return default
-	return "NON_DEFAULT_OS_ENV_VALUE"
+    print(f"Attempted to fetch os environment variable during parse, returning a mocked value for {key}")
+    if key == "JENKINS_HOME" and default is None:  # fix https://github.com/astronomer/astro-cli/issues/601
+        return None
+    if default:
+        return default
+    return "NON_DEFAULT_OS_ENV_VALUE"
 
 
 os.getenv = os_getenv_monkeypatch
 # # =========== /MONKEYPATCH OS.GETENV() ===========
 
 # =========== MONKEYPATCH VARIABLE.GET() ===========
 
+
 class magic_dict(dict):
-	def __init__(self,*args,**kwargs):
-		self.update(*args,**kwargs)
+    def __init__(self, *args, **kwargs):
+        self.update(*args, **kwargs)
 
-	def __getitem__(self,key):
-		return {}.get(key, 'MOCKED_KEY_VALUE')
+    def __getitem__(self, key):
+        return {}.get(key, "MOCKED_KEY_VALUE")
 
 
 def variable_get_monkeypatch(key: str, default_var=None, deserialize_json=False):
-	print(f"Attempted to get Variable value during parse, returning a mocked value for {key}")
+    print(f"Attempted to get Variable value during parse, returning a mocked value for {key}")
+
+    if default_var:
+        return default_var
+    if deserialize_json:
+        return magic_dict()
+    return "NON_DEFAULT_MOCKED_VARIABLE_VALUE"
 
-	if default_var:
-		return default_var
-	if deserialize_json:
-		return magic_dict()
-	return "NON_DEFAULT_MOCKED_VARIABLE_VALUE"
 
 Variable.get = variable_get_monkeypatch
 # # =========== /MONKEYPATCH VARIABLE.GET() ===========
 
 
 @contextmanager
 def suppress_logging(namespace):
-	"""
-	Suppress logging within a specific namespace to keep tests "clean" during build
-	"""
-	logger = logging.getLogger(namespace)
-	old_value = logger.disabled
-	logger.disabled = True
-	try:
-		yield
-	finally:
-		logger.disabled = old_value
+    """
+    Suppress logging within a specific namespace to keep tests "clean" during build
+    """
+    logger = logging.getLogger(namespace)
+    old_value = logger.disabled
+    logger.disabled = True
+    try:
+        yield
+    finally:
+        logger.disabled = old_value
 
-def get_import_errors():
-	"""
-	Generate a tuple for import errors in the dag bag
-	"""
-	with suppress_logging('airflow') :
-		dag_bag = DagBag(include_examples=False)
 
-		def strip_path_prefix(path):
-			return os.path.relpath(path ,os.environ.get('AIRFLOW_HOME'))
+def get_import_errors():
+    """
+    Generate a tuple for import errors in the dag bag
+    """
+    with suppress_logging("airflow"):
+        dag_bag = DagBag(include_examples=False)
 
+        def strip_path_prefix(path):
+            return os.path.relpath(path, os.environ.get("AIRFLOW_HOME"))
 
-		# we prepend "(None,None)" to ensure that a test object is always created even if its a no op.
-		return [(None,None)] +[ ( strip_path_prefix(k) , v.strip() ) for k,v in dag_bag.import_errors.items()]
+        # we prepend "(None,None)" to ensure that a test object is always created even if its a no op.
+        return [(None, None)] + [(strip_path_prefix(k), v.strip()) for k, v in dag_bag.import_errors.items()]
 
 
 @pytest.mark.parametrize("rel_path,rv", get_import_errors(), ids=[x[0] for x in get_import_errors()])
-def test_file_imports(rel_path,rv):
-	""" Test for import errors on a file """
-	if rel_path and rv : #Make sure our no op test doesn't raise an error
-		raise Exception(f"{rel_path} failed to import with message \n {rv}")
+def test_file_imports(rel_path, rv):
+    """Test for import errors on a file"""
+    if rel_path and rv:  # Make sure our no op test doesn't raise an error
+        raise Exception(f"{rel_path} failed to import with message \n {rv}")

airflow/.dockerignore

@@ -3,4 +3,4 @@ astro
 .env
 airflow_settings.yaml
 logs/
-dags/
+dags/

airflow/Dockerfile

@@ -1,3 +1,3 @@
 FROM quay.io/astronomer/astro-runtime:9.1.0-base
 
-COPY include/airflow_provider_weaviate-1.0.0-py3-none-any.whl /tmp
+COPY include/airflow_provider_weaviate-1.0.0-py3-none-any.whl /tmp