Merge pull request #322 from dgraeber/refactor/deployment-code

refactoring deployment_commands, adding unittests

CHANGELOG.md

@@ -12,8 +12,11 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 
 
 ### Changes
+- refactored deployment_commands
+- added python upper limit versioning (to `<3.12`)
 
 ### Fixes
+- typo in messaging about errored manifest
 
 ## v2.6.5 (2023-05-03)
 

seedfarmer/commands/_deployment_commands.py

@@ -124,7 +124,7 @@ def _process_data_files(data_files: List[DataFile], module_name: str, group_name
         for missing_file in missing_files:
             print(f"  {missing_file}")
         print_bolded(message="Exiting Deployment", color="red")
-        exit(1)
+        raise ValueError("Missing DataFiles - cannot process")
 
 
 def _execute_deploy(
@@ -257,12 +257,23 @@ def _execute_destroy(
     return resp
 
 
-def _deploy_deployment_is_not_dry_run(
+def _deploy_validated_deployment(
     deployment_manifest: DeploymentManifest,
     deployment_manifest_wip: DeploymentManifest,
     groups_to_deploy: List[ModulesManifest],
+    dryrun: bool,
 ) -> None:
     if groups_to_deploy:
+        if dryrun:
+            mods_would_deploy = [
+                (_module.target_account, _module.target_region, deployment_manifest.name, _group.name, _module.name)
+                for _group in groups_to_deploy
+                for _module in _group.modules
+            ]
+            _print_modules(
+                f"Modules scheduled to be deployed (created or updated): {deployment_manifest.name}", mods_would_deploy
+            )
+            return
         deployment_manifest_wip.groups = groups_to_deploy
         print_manifest_inventory(
             f"Modules scheduled to be deployed (created or updated): {deployment_manifest_wip.name}",
@@ -330,17 +341,6 @@ def _render_permissions_boundary_arn(
     du.write_deployed_deployment_manifest(deployment_manifest=deployment_manifest)
 
 
-def _deploy_deployment_is_dry_run(groups_to_deploy: List[ModulesManifest], deployment_name: str) -> None:
-    mods_would_deploy = []
-    if groups_to_deploy:
-        for _group in groups_to_deploy:
-            for _module in _group.modules:
-                mods_would_deploy.append(
-                    [_module.target_account, _module.target_region, deployment_name, _group.name, _module.name]
-                )
-    _print_modules(f"Modules scheduled to be deployed (created or updated): {deployment_name}", mods_would_deploy)
-
-
 def prime_target_accounts(deployment_manifest: DeploymentManifest) -> None:
     # TODO: Investigate whether we need to validate the requested mappings against previously deployed mappings
 
@@ -615,23 +615,17 @@ def deploy_deployment(
                     name=group.name, path=group.path, concurrency=group.concurrency, modules=modules_to_deploy
                 )
             )
-    if unchanged_modules:
-        _print_modules(
-            f"Modules deployed that are up to date (will not be changed): {deployment_name} ", unchanged_modules
-        )
-    if not dryrun:
-        _deploy_deployment_is_not_dry_run(
-            deployment_manifest=deployment_manifest,
-            deployment_manifest_wip=deployment_manifest_wip,
-            groups_to_deploy=groups_to_deploy,
-        )
-    else:
-        _deploy_deployment_is_dry_run(groups_to_deploy=groups_to_deploy, deployment_name=deployment_name)
-
+    _print_modules(
+        f"Modules deployed that are up to date (will not be changed): {deployment_name} ", unchanged_modules
+    ) if unchanged_modules else None
+    _deploy_validated_deployment(
+        deployment_manifest=deployment_manifest,
+        deployment_manifest_wip=deployment_manifest_wip,
+        groups_to_deploy=groups_to_deploy,
+        dryrun=dryrun,
+    )
     print_bolded(f"To see all deployed modules, run seedfarmer list modules -d {deployment_name}")
-
-    if show_manifest:
-        print_manifest_json(deployment_manifest)
+    print_manifest_json(deployment_manifest) if show_manifest else None
 
 
 def apply(
@@ -715,7 +709,7 @@ def apply(
                 _logger.error(e)
                 _logger.error(f"Cannot parse a file at {os.path.join(config.OPS_ROOT, module_group.path)}")
                 _logger.error("Verify that elements are filled out and yaml compliant")
-                exit(1)
+                raise ValueError("Cannot parse manifest file path")
     deployment_manifest.validate_and_set_module_defaults()
 
     prime_target_accounts(deployment_manifest=deployment_manifest)
@@ -732,7 +726,7 @@ def apply(
             header_message="The following modules requested for destroy have dependencies that prevent destruction:",
             errored_list=violations,
         )
-        exit(1)
+        raise Exception("Modules cannot be destroyed due to dependencies")
 
     destroy_deployment(
         destroy_manifest=destroy_manifest,

seedfarmer/mgmt/deploy_utils.py

@@ -173,7 +173,7 @@ def validate_group_parameters(group: ModulesManifest) -> None:
           No module can refer to its own group for parameter lookups!!
         """
         print_bolded(message=message, color="red")
-        exit(1)
+        raise ValueError(message)
 
 
 def validate_module_dependencies(

setup.py

@@ -43,7 +43,7 @@
     license=about["__license__"],
     packages=find_packages(include=["seed-farmer", "seedfarmer", "seedfarmer.*", "seed-farmer.*"]),
     keywords=["aws", "cdk"],
-    python_requires=">=3.7",
+    python_requires=">=3.7,<3.12",
     install_requires=[
         "aws-codeseeder~=0.9.0",
         "cookiecutter~=2.1.0",

test/unit-test/test_commands_deployment.py

@@ -6,7 +6,7 @@
 import pytest
 import seedfarmer.commands._deployment_commands as dc
 
-from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModuleParameter
+from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModuleParameter, DataFile
 from seedfarmer.models._deploy_spec import DeploySpec
 from seedfarmer.services._service_utils import boto3_client
 from seedfarmer.services.session_manager import SessionManager
@@ -72,9 +72,8 @@ def test_apply_violations(session_manager,mocker):
     mocker.patch("seedfarmer.commands._deployment_commands.du.filter_deploy_destroy", return_value=None)
     mocker.patch("seedfarmer.commands._deployment_commands.write_deployment_manifest", return_value=None)
     mocker.patch("seedfarmer.commands._deployment_commands.du.validate_module_dependencies", return_value=[{"module1":["moduleA","moduleB"]}])
-    with pytest.raises(SystemExit) as pytest_wrapped_e:
+    with pytest.raises(Exception):
         dc.apply(deployment_manifest_path="test/unit-test/mock_data/manifests/module-test/deployment-hc.yaml")
-    assert pytest_wrapped_e.type == SystemExit
 
 
 @pytest.mark.commands
@@ -109,14 +108,7 @@ def test_destroy_not_found(session_manager,mocker):
 #     #     teardown.
     ### COMMENT....I cannot get the nested threads to mock
 #     dc.tear_down_target_accounts(deployment_manifest=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest))
-
-
-@pytest.mark.commands
-@pytest.mark.commands_deployment
-def test_deploy_deployment_is_dry_run(session_manager,mocker):
-    dep = DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest)
-    dc._deploy_deployment_is_dry_run(groups_to_deploy=dep.groups, deployment_name="myapp") 
-
+
 
 @pytest.mark.commands
 @pytest.mark.commands_deployment
@@ -126,6 +118,32 @@ def test_clone_module_repo(mocker):
     # Rerun it so all methods are hit
     return_dir =dc._clone_module_repo(git_path=git_path_test)
 
+
+@pytest.mark.commands
+@pytest.mark.commands_deployment
+def test_process_data_files(mocker):
+    mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo",return_value=("git","path"))
+    mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files",return_value=[])
+    git_path_test = "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1"
+    datafile_list =[]
+    datafile_list.append(DataFile(file_path=git_path_test))
+    datafile_list.append(DataFile(file_path=""))
+
+    dc._process_data_files(data_files=datafile_list, module_name="test",group_name="test")
+
+
+@pytest.mark.commands
+@pytest.mark.commands_deployment
+def test_process_data_files_error(mocker):
+    mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo",return_value=("git","path"))
+    mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files",return_value=["hey"])
+    git_path_test = "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1"
+    datafile_list =[]
+    datafile_list.append(DataFile(file_path=git_path_test))
+    datafile_list.append(DataFile(file_path=""))
+    with pytest.raises(ValueError):
+        dc._process_data_files(data_files=datafile_list, module_name="test",group_name="test")
+
 
 
 @pytest.mark.commands
@@ -232,7 +250,7 @@ def test_deploy_deployment(session_manager,mocker):
     mocker.patch("seedfarmer.commands._deployment_commands.hashlib.md5",return_value=mock_hashlib)
 
 
-    mocker.patch("seedfarmer.commands._deployment_commands._deploy_deployment_is_not_dry_run",return_value=None)
+    mocker.patch("seedfarmer.commands._deployment_commands._deploy_validated_deployment",return_value=None)
     mocker.patch("seedfarmer.commands._deployment_commands.du.need_to_build",
                  return_value=None)
     # mocker.patch("seedfarmer.commands._deployment_commands.module_info_index.get_module_info",

test/unit-test/test_mgmt_deploy_utils.py

@@ -19,7 +19,7 @@
 
 import pytest
 import seedfarmer.mgmt.deploy_utils as du
-from seedfarmer.models.manifests import DeploymentManifest, ModulesManifest
+from seedfarmer.models.manifests import DeploymentManifest, ModulesManifest, DataFile
 from seedfarmer.services._service_utils import boto3_client
 from seedfarmer.services.session_manager import SessionManager
 import mock_data.mock_manifests as mock_manifests
@@ -80,11 +80,8 @@ def test_validate_group_parameters():
 @pytest.mark.mgmt_deployment_utils
 def test_validate_group_parameters_failure():
     manifest = ModulesManifest(**mock_manifests.modules_manifest_duplicate)
-    with pytest.raises(SystemExit) as dupe_error:
-        du.validate_group_parameters(manifest)
-
-    assert dupe_error.value.code == 1
-
+    with pytest.raises(ValueError):
+        du.validate_group_parameters(manifest)    
 
 @pytest.mark.mgmt
 @pytest.mark.mgmt_deployment_utils
@@ -260,4 +257,11 @@ def test_validate_module_dependencies():
     du.validate_module_dependencies(module_dependencies=mock_deployment_manifest_for_destroy.module_dependencies,
                                     destroy_manifest=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest)
 
-    )
+    )
+
+@pytest.mark.mgmt
+@pytest.mark.mgmt_deployment_utils_filter   
+def test_validate_datafiles():
+
+    files = DataFile(file_path="")
+    du.validate_data_files(data_files=[files])