Merge pull request #2444 from bagerard/improve_code_health

Improve code health

Author: bagerard

.pre-commit-config.yaml

@@ -1,12 +1,17 @@
 fail_fast: false
 repos:
-    - repo: https://github.com/ambv/black
-      rev: 19.10b0
-      hooks:
-        - id: black
-    - repo: https://gitlab.com/pycqa/flake8
-      rev: 3.8.0a2
-      hooks:
-        - id: flake8
-          additional_dependencies:
-            - flake8-import-order
+  - repo: https://github.com/ambv/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-import-order
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.4
+    hooks:
+      - id: pyupgrade
+        args: [--py36-plus]

.travis.yml

@@ -16,11 +16,11 @@
 language: python
 dist: xenial
 python:
-- 3.6
-- 3.7
-- 3.8
-- 3.9
-- pypy3
+  - 3.6
+  - 3.7
+  - 3.8
+  - 3.9
+  - pypy3
 
 env:
   global:
@@ -42,14 +42,14 @@ matrix:
   fast_finish: true
 
   include:
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
-  - python: 3.7
-    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_11}
-  - python: 3.8
-    env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_11}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
+    - python: 3.7
+      env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_11}
+    - python: 3.8
+      env: MONGODB=${MONGODB_4_0} PYMONGO=${PYMONGO_3_11}
 
 install:
   # Install Mongo
@@ -69,7 +69,7 @@ before_script:
   - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
   # Run pre-commit hooks (black, flake8, etc) on entire codebase
   - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
-  - mongo --eval 'db.version();'    # Make sure mongo is awake
+  - mongo --eval 'db.version();' # Make sure mongo is awake
 
 script:
   - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
@@ -83,8 +83,8 @@ notifications:
 # Only run builds on the master branch and GitHub releases (tagged as vX.Y.Z)
 branches:
   only:
-  - master
-  - /^v.*$/
+    - master
+    - /^v.*$/
 
 # Whenever a new release is created via GitHub, publish it on PyPI.
 deploy:

benchmarks/test_basic_doc_ops.py

@@ -45,7 +45,7 @@ def init_book():
 
     print(
         "Doc setattr: %.3fus"
-        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)
+        % (timeit(lambda: setattr(b, "name", "New name"), 10000) * 10 ** 6)  # noqa B010
     )
 
     print("Doc to mongo: %.3fus" % (timeit(b.to_mongo, 1000) * 10 ** 6))

docs/conf.py

@@ -41,8 +41,8 @@
 master_doc = "index"
 
 # General information about the project.
-project = u"MongoEngine"
-copyright = u"2009, MongoEngine Authors"
+project = "MongoEngine"
+copyright = "2009, MongoEngine Authors"  # noqa: A001
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

mongoengine/__init__.py

@@ -10,12 +10,12 @@
 # mongoengine, e.g. instead of `from mongoengine.connection import connect`,
 # users can simply use `from mongoengine import connect`, or even
 # `from mongoengine import *` and then `connect('testdb')`.
-from mongoengine.connection import *
-from mongoengine.document import *
-from mongoengine.errors import *
-from mongoengine.fields import *
-from mongoengine.queryset import *
-from mongoengine.signals import *
+from mongoengine.connection import *  # noqa: F401
+from mongoengine.document import *  # noqa: F401
+from mongoengine.errors import *  # noqa: F401
+from mongoengine.fields import *  # noqa: F401
+from mongoengine.queryset import *  # noqa: F401
+from mongoengine.signals import *  # noqa: F401
 
 
 __all__ = (

mongoengine/base/datastructures.py

@@ -67,11 +67,11 @@ def __getitem__(self, key):
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -97,7 +97,7 @@ def __setstate__(self, state):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
             if key:
-                self._instance._mark_as_changed("{}.{}".format(self._name, key))
+                self._instance._mark_as_changed(f"{self._name}.{key}")
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -133,12 +133,12 @@ def __getitem__(self, key):
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             # Replace dict by BaseDict
-            value = BaseDict(value, None, "{}.{}".format(self._name, key))
+            value = BaseDict(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
             # Replace list by BaseList
-            value = BaseList(value, None, "{}.{}".format(self._name, key))
+            value = BaseList(value, None, f"{self._name}.{key}")
             super().__setitem__(key, value)
             value._instance = self._instance
         return value
@@ -180,9 +180,7 @@ def __setitem__(self, key, value):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, "_mark_as_changed"):
             if key is not None:
-                self._instance._mark_as_changed(
-                    "{}.{}".format(self._name, key % len(self))
-                )
+                self._instance._mark_as_changed(f"{self._name}.{key % len(self)}")
             else:
                 self._instance._mark_as_changed(self._name)
 
@@ -429,7 +427,7 @@ class SpecificStrictDict(cls):
 
                 def __repr__(self):
                     return "{%s}" % ", ".join(
-                        '"{!s}": {!r}'.format(k, v) for k, v in self.items()
+                        f'"{k!s}": {v!r}' for k, v in self.items()
                     )
 
             cls._classes[allowed_keys] = SpecificStrictDict
@@ -472,4 +470,4 @@ def __getattr__(self, name):
             raise AttributeError()
 
     def __repr__(self):
-        return "<LazyReference({}, {!r})>".format(self.document_type, self.pk)
+        return f"<LazyReference({self.document_type}, {self.pk!r})>"

mongoengine/base/document.py

@@ -89,9 +89,7 @@ def __init__(self, *args, **values):
                 list(self._fields.keys()) + ["id", "pk", "_cls", "_text_score"]
             )
             if _undefined_fields:
-                msg = ('The fields "{}" do not exist on the document "{}"').format(
-                    _undefined_fields, self._class_name
-                )
+                msg = f'The fields "{_undefined_fields}" do not exist on the document "{self._class_name}"'
                 raise FieldDoesNotExist(msg)
 
         if self.STRICT and not self._dynamic:
@@ -231,10 +229,10 @@ def __setstate__(self, data):
                 setattr(self, k, data[k])
         if "_fields_ordered" in data:
             if self._dynamic:
-                setattr(self, "_fields_ordered", data["_fields_ordered"])
+                self._fields_ordered = data["_fields_ordered"]
             else:
                 _super_fields_ordered = type(self)._fields_ordered
-                setattr(self, "_fields_ordered", _super_fields_ordered)
+                self._fields_ordered = _super_fields_ordered
 
         dynamic_fields = data.get("_dynamic_fields") or SON()
         for k in dynamic_fields.keys():
@@ -244,8 +242,7 @@ def __iter__(self):
         return iter(self._fields_ordered)
 
     def __getitem__(self, name):
-        """Dictionary-style field access, return a field's value if present.
-        """
+        """Dictionary-style field access, return a field's value if present."""
         try:
             if name in self._fields_ordered:
                 return getattr(self, name)
@@ -254,8 +251,7 @@ def __getitem__(self, name):
         raise KeyError(name)
 
     def __setitem__(self, name, value):
-        """Dictionary-style field access, set a field's value.
-        """
+        """Dictionary-style field access, set a field's value."""
         # Ensure that the field exists before settings its value
         if not self._dynamic and name not in self._fields:
             raise KeyError(name)
@@ -277,7 +273,7 @@ def __repr__(self):
         except (UnicodeEncodeError, UnicodeDecodeError):
             u = "[Bad Unicode data]"
         repr_type = str if u is None else type(u)
-        return repr_type("<{}: {}>".format(self.__class__.__name__, u))
+        return repr_type(f"<{self.__class__.__name__}: {u}>")
 
     def __str__(self):
         # TODO this could be simpler?
@@ -433,7 +429,7 @@ def validate(self, clean=True):
                 pk = self.pk
             elif self._instance and hasattr(self._instance, "pk"):
                 pk = self._instance.pk
-            message = "ValidationError ({}:{}) ".format(self._class_name, pk)
+            message = f"ValidationError ({self._class_name}:{pk}) "
             raise ValidationError(message, errors=errors)
 
     def to_json(self, *args, **kwargs):
@@ -506,7 +502,7 @@ def _mark_as_changed(self, key):
         if "." in key:
             key, rest = key.split(".", 1)
             key = self._db_field_map.get(key, key)
-            key = "{}.{}".format(key, rest)
+            key = f"{key}.{rest}"
         else:
             key = self._db_field_map.get(key, key)
 
@@ -578,7 +574,7 @@ def _nestable_types_clear_changed_fields(data):
         else:
             iterator = data.items()
 
-        for index_or_key, value in iterator:
+        for _index_or_key, value in iterator:
             if hasattr(value, "_get_changed_fields") and not isinstance(
                 value, Document
             ):  # don't follow references
@@ -602,23 +598,22 @@ def _nestable_types_changed_fields(changed_fields, base_key, data):
             iterator = data.items()
 
         for index_or_key, value in iterator:
-            item_key = "{}{}.".format(base_key, index_or_key)
+            item_key = f"{base_key}{index_or_key}."
             # don't check anything lower if this key is already marked
             # as changed.
             if item_key[:-1] in changed_fields:
                 continue
 
             if hasattr(value, "_get_changed_fields"):
                 changed = value._get_changed_fields()
-                changed_fields += ["{}{}".format(item_key, k) for k in changed if k]
+                changed_fields += [f"{item_key}{k}" for k in changed if k]
             elif isinstance(value, (list, tuple, dict)):
                 BaseDocument._nestable_types_changed_fields(
                     changed_fields, item_key, value
                 )
 
     def _get_changed_fields(self):
-        """Return a list of all fields that have explicitly been changed.
-        """
+        """Return a list of all fields that have explicitly been changed."""
         EmbeddedDocument = _import_class("EmbeddedDocument")
         ReferenceField = _import_class("ReferenceField")
         GenericReferenceField = _import_class("GenericReferenceField")
@@ -643,7 +638,7 @@ def _get_changed_fields(self):
             if isinstance(data, EmbeddedDocument):
                 # Find all embedded fields that have been changed
                 changed = data._get_changed_fields()
-                changed_fields += ["{}{}".format(key, k) for k in changed if k]
+                changed_fields += [f"{key}{k}" for k in changed if k]
             elif isinstance(data, (list, tuple, dict)):
                 if hasattr(field, "field") and isinstance(
                     field.field, (ReferenceField, GenericReferenceField)
@@ -750,8 +745,7 @@ def _get_collection_name(cls):
 
     @classmethod
     def _from_son(cls, son, _auto_dereference=True, created=False):
-        """Create an instance of a Document (subclass) from a PyMongo SON (dict)
-        """
+        """Create an instance of a Document (subclass) from a PyMongo SON (dict)"""
         if son and not isinstance(son, dict):
             raise ValueError(
                 "The source SON object needs to be of type 'dict' but a '%s' was found"
@@ -796,11 +790,10 @@ def _from_son(cls, son, _auto_dereference=True, created=False):
                     errors_dict[field_name] = e
 
         if errors_dict:
-            errors = "\n".join(
-                ["Field '{}' - {}".format(k, v) for k, v in errors_dict.items()]
-            )
+            errors = "\n".join([f"Field '{k}' - {v}" for k, v in errors_dict.items()])
             msg = "Invalid data to create a `{}` instance.\n{}".format(
-                cls._class_name, errors,
+                cls._class_name,
+                errors,
             )
             raise InvalidDocumentError(msg)
 
@@ -968,10 +961,7 @@ def _unique_with_indexes(cls, namespace=""):
                     unique_fields += unique_with
 
                 # Add the new index to the list
-                fields = [
-                    ("{}{}".format(namespace, f), pymongo.ASCENDING)
-                    for f in unique_fields
-                ]
+                fields = [(f"{namespace}{f}", pymongo.ASCENDING) for f in unique_fields]
                 index = {"fields": fields, "unique": True, "sparse": sparse}
                 unique_indexes.append(index)
 
@@ -1007,9 +997,7 @@ def _geo_indices(cls, inspected=None, parent_field=None):
             "PolygonField",
         )
 
-        geo_field_types = tuple(
-            [_import_class(field) for field in geo_field_type_names]
-        )
+        geo_field_types = tuple(_import_class(field) for field in geo_field_type_names)
 
         for field in cls._fields.values():
             if not isinstance(field, geo_field_types):
@@ -1027,7 +1015,7 @@ def _geo_indices(cls, inspected=None, parent_field=None):
             elif field._geo_index:
                 field_name = field.db_field
                 if parent_field:
-                    field_name = "{}.{}".format(parent_field, field_name)
+                    field_name = f"{parent_field}.{field_name}"
                 geo_indices.append({"fields": [(field_name, field._geo_index)]})
 
         return geo_indices
@@ -1165,8 +1153,7 @@ def _lookup_field(cls, parts):
 
     @classmethod
     def _translate_field_name(cls, field, sep="."):
-        """Translate a field attribute name to a database field name.
-        """
+        """Translate a field attribute name to a database field name."""
         parts = field.split(sep)
         parts = [f.db_field for f in cls._lookup_field(parts)]
         return ".".join(parts)