Merge pull request #226 from networktocode/release-1.8.0

Release 1.8.0

Author: glennmatthews

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -40,7 +40,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
     needs:
@@ -53,7 +53,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Linting: pydocstyle"
         run: "poetry run invoke pydocstyle"
     needs:
@@ -66,7 +66,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
     needs:
@@ -79,8 +79,8 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
-      - name: "Linting: flake8"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
+      - name: "Linting: mypy"
         run: "poetry run invoke mypy"
     needs:
       - "black"
@@ -92,7 +92,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
     needs:
@@ -103,7 +103,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Build Container"
         run: "poetry run invoke build"
     needs:
@@ -118,7 +118,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v1"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
       - name: "Build Container"
         run: "poetry run invoke build"
       - name: "Linting: Pylint"
@@ -137,7 +137,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v5"
         with:
           python-version: "${{ matrix.python-version }}"
       - name: "Install redis"
@@ -203,3 +203,39 @@ jobs:
           password: "${{ secrets.PYPI_API_TOKEN }}"
     needs:
       - "unittest"
+  slack-notify:
+    needs:
+      - "publish_gh"
+      - "publish_pypi"
+    name: "Send notification to the Slack"
+    runs-on: "ubuntu-20.04"
+    env:
+      SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+      SLACK_MESSAGE: >-
+        *NOTIFICATION: NEW-RELEASE-PUBLISHED*\n
+        Repository: <${{ github.server_url }}/${{ github.repository }}|${{ github.repository }}>\n
+        Release: <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>\n
+        Published by: <${{ github.server_url }}/${{ github.actor }}|${{ github.actor }}>
+    steps:
+      - name: "Send a notification to Slack"
+        # ENVs cannot be used directly in job.if. This is a workaround to check
+        # if SLACK_WEBHOOK_URL is present.
+        if: "${{ env.SLACK_WEBHOOK_URL != '' }}"
+        uses: "slackapi/[email protected]"
+        with:
+          payload: |
+            {
+              "text": "${{ env.SLACK_MESSAGE }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "${{ env.SLACK_MESSAGE }}"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+          SLACK_WEBHOOK_TYPE: "INCOMING_WEBHOOK"

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v1.8.0 - 2023-04-18
+
+### Added
+
+- #182 - Added `get_or_add_model_instance()` and `update_or_add_model_instance()` APIs.
+- #189 - Added note in `README.md` about running `invoke tests`.
+- #190 - Added note in `README.md` about running `invoke build`.
+
+### Changed
+
+- #77/#188 - `sync_from()` and `sync_to()` now return the `Diff` that was applied.
+- #211 - Loosened `packaging` and `structlog` library dependency constraints for broader compatibility.
+
 ## v1.7.0 - 2022-11-03
 
 ### Changed
@@ -10,7 +23,7 @@
 ### Added
 
 - #174 - Add methods to load data from dictionary and enable tree traversal
-- #174 - Add a get_or_none method to the DiffSync class
+- #174 - Add a `get_or_none` method to the DiffSync class
 - #168 - Add 'skip' counter to diff.summary()
 - #169/#170 - Add documentation about model processing order
 - #121/#140 - Add and configure renovate

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 Network to Code <[email protected]>
+Copyright 2020-2023 Network to Code <[email protected]>
 Network to Code, LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");

README.md

@@ -75,5 +75,8 @@ The project is following Network to Code software development guidelines and are
 - Black, Pylint, Bandit, flake8, and pydocstyle, mypy for Python linting, formatting and type hint checking.
 - pytest, coverage, and unittest for unit tests.
 
+You can ensure your contribution adheres to these checks by running `invoke tests` from the CLI.
+The command `invoke build` builds a docker container with all the necessary dependencies (including the redis backend) locally to facilitate the execution of these tests.
+
 # Questions
 Please see the [documentation](https://diffsync.readthedocs.io/en/latest/index.html) for detailed documentation on how to use `diffsync`. For any additional questions or comments, feel free to swing by the [Network to Code slack channel](https://networktocode.slack.com/) (channel #networktocode). Sign up [here](http://slack.networktocode.com/)

diffsync/__init__.py

@@ -526,14 +526,14 @@ def load_from_dict(self, data: Dict):
     # Synchronization between DiffSync instances
     # ------------------------------------------------------------------------------
 
-    def sync_from(
+    def sync_from(  # pylint: disable=too-many-arguments
         self,
         source: "DiffSync",
         diff_class: Type[Diff] = Diff,
         flags: DiffSyncFlags = DiffSyncFlags.NONE,
         callback: Optional[Callable[[Text, int, int], None]] = None,
         diff: Optional[Diff] = None,
-    ):  # pylint: disable=too-many-arguments:
+    ) -> Diff:
         """Synchronize data from the given source DiffSync object into the current DiffSync object.
 
         Args:
@@ -543,6 +543,10 @@ def sync_from(
             callback (function): Function with parameters (stage, current, total), to be called at intervals as the
                 calculation of the diff and subsequent sync proceed.
             diff (Diff): An existing diff to be used rather than generating a completely new diff.
+        Returns:
+            Diff: Diff between origin object and source
+        Raises:
+            DiffClassMismatch: The provided diff's class does not match the diff_class
         """
         if diff_class and diff:
             if not isinstance(diff, diff_class):
@@ -558,14 +562,16 @@ def sync_from(
         if result:
             self.sync_complete(source, diff, flags, syncer.base_logger)
 
-    def sync_to(
+        return diff
+
+    def sync_to(  # pylint: disable=too-many-arguments
         self,
         target: "DiffSync",
         diff_class: Type[Diff] = Diff,
         flags: DiffSyncFlags = DiffSyncFlags.NONE,
         callback: Optional[Callable[[Text, int, int], None]] = None,
         diff: Optional[Diff] = None,
-    ):  # pylint: disable=too-many-arguments
+    ) -> Diff:
         """Synchronize data from the current DiffSync object into the given target DiffSync object.
 
         Args:
@@ -575,15 +581,19 @@ def sync_to(
             callback (function): Function with parameters (stage, current, total), to be called at intervals as the
                 calculation of the diff and subsequent sync proceed.
             diff (Diff): An existing diff that will be used when determining what needs to be synced.
+        Returns:
+            Diff: Diff between origin object and target
+        Raises:
+            DiffClassMismatch: The provided diff's class does not match the diff_class
         """
-        target.sync_from(self, diff_class=diff_class, flags=flags, callback=callback, diff=diff)
+        return target.sync_from(self, diff_class=diff_class, flags=flags, callback=callback, diff=diff)
 
     def sync_complete(
         self,
         source: "DiffSync",
         diff: Diff,
         flags: DiffSyncFlags = DiffSyncFlags.NONE,
-        logger: structlog.BoundLogger = None,
+        logger: Optional[structlog.BoundLogger] = None,
     ):
         """Callback triggered after a `sync_from` operation has completed and updated the model data of this instance.
 
@@ -776,7 +786,7 @@ def remove(self, obj: DiffSyncModel, remove_children: bool = False):
         return self.store.remove(obj=obj, remove_children=remove_children)
 
     def get_or_instantiate(
-        self, model: Type[DiffSyncModel], ids: Dict, attrs: Dict = None
+        self, model: Type[DiffSyncModel], ids: Dict, attrs: Optional[Dict] = None
     ) -> Tuple[DiffSyncModel, bool]:
         """Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs.
 
@@ -790,19 +800,41 @@ def get_or_instantiate(
         """
         return self.store.get_or_instantiate(model=model, ids=ids, attrs=attrs)
 
+    def get_or_add_model_instance(self, obj: DiffSyncModel) -> Tuple[DiffSyncModel, bool]:
+        """Attempt to get the object with provided obj identifiers or instantiate obj.
+
+        Args:
+            obj: An obj of the DiffSyncModel to get or add.
+
+        Returns:
+            Provides the existing or new object and whether it was created or not.
+        """
+        return self.store.get_or_add_model_instance(obj=obj)
+
     def update_or_instantiate(self, model: Type[DiffSyncModel], ids: Dict, attrs: Dict) -> Tuple[DiffSyncModel, bool]:
         """Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs.
 
         Args:
-            model (DiffSyncModel): The DiffSyncModel to get or create.
-            ids (Dict): Identifiers for the DiffSyncModel to get or create with.
+            model (DiffSyncModel): The DiffSyncModel to update or create.
+            ids (Dict): Identifiers for the DiffSyncModel to update or create with.
             attrs (Dict): Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs.
 
         Returns:
             Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not.
         """
         return self.store.update_or_instantiate(model=model, ids=ids, attrs=attrs)
 
+    def update_or_add_model_instance(self, obj: DiffSyncModel) -> Tuple[DiffSyncModel, bool]:
+        """Attempt to update an existing object with provided obj ids/attrs or instantiate obj.
+
+        Args:
+            instance: An instance of the DiffSyncModel to update or create.
+
+        Returns:
+            Provides the existing or new object and whether it was created or not.
+        """
+        return self.store.update_or_add_model_instance(obj=obj)
+
     def count(self, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], None] = None):
         """Count how many objects of one model type exist in the backend store.
 

diffsync/helpers.py

@@ -328,7 +328,7 @@ def perform_sync(self) -> bool:
         self.base_logger.info("Sync complete")
         return changed
 
-    def sync_diff_element(self, element: DiffElement, parent_model: "DiffSyncModel" = None) -> bool:
+    def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSyncModel"] = None) -> bool:
         """Recursively synchronize the given DiffElement and its children, if any, into the dst_diffsync.
 
         Helper method to `perform_sync`.

diffsync/store/__init__.py

@@ -134,7 +134,7 @@ def count(self, *, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"], No
         raise NotImplementedError
 
     def get_or_instantiate(
-        self, *, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict = None
+        self, *, model: Type["DiffSyncModel"], ids: Dict, attrs: Optional[Dict] = None
     ) -> Tuple["DiffSyncModel", bool]:
         """Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs.
 
@@ -159,6 +159,24 @@ def get_or_instantiate(
 
         return obj, created
 
+    def get_or_add_model_instance(self, obj: "DiffSyncModel") -> Tuple["DiffSyncModel", bool]:
+        """Attempt to get the object with provided obj identifiers or instantiate obj.
+
+        Args:
+            obj: An obj of the DiffSyncModel to get or add.
+
+        Returns:
+            Provides the existing or new object and whether it was added or not.
+        """
+        model = obj.get_type()
+        ids = obj.get_unique_id()
+
+        try:
+            return self.get(model=model, identifier=ids), False
+        except ObjectNotFound:
+            self.add(obj=obj)
+            return obj, True
+
     def update_or_instantiate(
         self, *, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict
     ) -> Tuple["DiffSyncModel", bool]:
@@ -188,6 +206,33 @@ def update_or_instantiate(
 
         return obj, created
 
+    def update_or_add_model_instance(self, obj: "DiffSyncModel") -> Tuple["DiffSyncModel", bool]:
+        """Attempt to update an existing object with provided ids/attrs or instantiate obj.
+
+        Args:
+            instance: An instance of the DiffSyncModel to update or create.
+
+        Returns:
+            Provides the existing or new object and whether it was added or not.
+        """
+        model = obj.get_type()
+        ids = obj.get_unique_id()
+        attrs = obj.get_attrs()
+
+        added = False
+        try:
+            obj = self.get(model=model, identifier=ids)
+        except ObjectNotFound:
+            # Add the object to the diffsync instance
+            self.add(obj=obj)
+            added = True
+
+        # Update existing obj with attrs
+        for attr, value in attrs.items():
+            setattr(obj, attr, value)
+
+        return obj, added
+
     def _get_object_class_and_model(
         self, model: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]
     ) -> Tuple[Union["DiffSyncModel", Type["DiffSyncModel"], None], str]:

examples/03-remote-system/diff.py

@@ -9,7 +9,6 @@ class AlphabeticalOrderDiff(Diff):
     def order_children_default(cls, children):
         """Simple diff to return all children in alphabetical order."""
         for child in sorted(children.values()):
-
             # it's possible to access additional information about the object
             #  like  child.action can be "update", "create" or "delete"
 

examples/03-remote-system/local_adapter.py

@@ -43,7 +43,6 @@ def load(self, filename=COUNTRIES_FILE):  # pylint: disable=arguments-differ
         # A Country object will be created for each country, it will be stored inside the adapter with self.add(),
         # and it will be linked to its parent with parent.add_child(item)
         for country in countries:
-
             # Retrive the parent region object from the internal cache.
             region = self.get(obj=self.region, identifier=slugify(country.get("region")))