Apply caching when handling parameters (#790)

Fixes: #781

## Why?

When generating the pipeline definition, it needs to know whether or not empty
targets are allowed, a config that is managed from the adfconfig.yml.
To support this, the Target instances fetch the configuration value from
parameter store.

When hundreds of targets are initialized, it will attempt to fetch this
parameter way too often. Especially considering the value is not likely
to change.

## What?

Apply caching on the fetch, put, delete operations in the Parameter Store
class. Such that all ADF components that need to perform lookups frequently
can benefit from this enhancement.

Author: sbkok

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cache.py

@@ -23,3 +23,7 @@ def get(self, key):
 
     def add(self, key, value):
         self._stash[key] = value
+
+    def remove(self, key):
+        if key in self._stash:
+            del self._stash[key]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py

@@ -7,13 +7,14 @@
 from botocore.config import Config
 
 # ADF imports
+from cache import Cache
 from errors import ParameterNotFoundError
 from paginator import paginator
 from logger import configure_logger
 
 LOGGER = configure_logger(__name__)
-PARAMETER_DESCRIPTION = 'DO NOT EDIT - Used by The AWS Deployment Framework'
-PARAMETER_PREFIX = '/adf'
+PARAMETER_DESCRIPTION = "DO NOT EDIT - Used by The AWS Deployment Framework"
+PARAMETER_PREFIX = "/adf"
 SSM_CONFIG = Config(
     retries={
         "max_attempts": 10,
@@ -22,99 +23,104 @@
 
 
 class ParameterStore:
-    """Class used for modeling Parameters
-    """
+    """Class used for modeling Parameters"""
 
-    def __init__(self, region, role):
-        self.client = role.client('ssm', region_name=region, config=SSM_CONFIG)
+    def __init__(self, region, role, cache=None):
+        self.cache = cache or Cache()
+        self.client = role.client("ssm", region_name=region, config=SSM_CONFIG)
 
-    def put_parameter(self, name, value, tier='Standard'):
-        """Puts a Parameter into Parameter Store
-        """
+    def put_parameter(self, name, value, tier="Standard"):
+        """Puts a Parameter into Parameter Store"""
         try:
             current_value = self.fetch_parameter(name)
             assert current_value == value
             LOGGER.debug(
-                'No need to update parameter %s with value %s since they '
-                'are the same',
+                "No need to update parameter %s with value %s since they "
+                "are the same",
                 ParameterStore._build_param_name(name),
                 value,
             )
         except (ParameterNotFoundError, AssertionError):
             param_name = ParameterStore._build_param_name(name)
             LOGGER.debug(
-                'Putting SSM Parameter %s with value %s',
+                "Putting SSM Parameter %s with value %s",
                 param_name,
                 value,
             )
             self.client.put_parameter(
                 Name=param_name,
                 Description=PARAMETER_DESCRIPTION,
                 Value=value,
-                Type='String',
+                Type="String",
                 Overwrite=True,
-                Tier=tier
+                Tier=tier,
             )
+            self.cache.add(param_name, value)
 
     def delete_parameter(self, name):
         param_name = ParameterStore._build_param_name(name)
         try:
-            LOGGER.debug('Deleting Parameter %s', param_name)
+            LOGGER.debug("Deleting Parameter %s", param_name)
+            self.cache.remove(param_name)
             self.client.delete_parameter(
                 Name=param_name,
             )
         except self.client.exceptions.ParameterNotFound:
             LOGGER.debug(
-                'Attempted to delete Parameter %s but it was not found',
+                "Attempted to delete Parameter %s but it was not found",
                 param_name,
             )
 
     def fetch_parameters_by_path(self, path):
-        """Gets a Parameter(s) by Path from Parameter Store (Recursively)
-        """
+        """Gets a Parameter(s) by Path from Parameter Store (Recursively)"""
         param_path = ParameterStore._build_param_name(path)
         try:
             LOGGER.debug(
-                'Fetching Parameters from path %s',
+                "Fetching Parameters from path %s",
                 param_path,
             )
             return paginator(
                 self.client.get_parameters_by_path,
                 Path=param_path,
                 Recursive=True,
-                WithDecryption=False
+                WithDecryption=False,
             )
         except self.client.exceptions.ParameterNotFound as error:
             raise ParameterNotFoundError(
-                f'Parameter Path {param_path} Not Found',
+                f"Parameter Path {param_path} Not Found",
             ) from error
 
     @staticmethod
     def _build_param_name(name, adf_only=True):
-        slash_name = name if name.startswith('/') else f"/{name}"
-        add_prefix = (
-            adf_only
-            and not slash_name.startswith(f"{PARAMETER_PREFIX}/")
-        )
-        param_prefix = PARAMETER_PREFIX if add_prefix else ''
+        slash_name = name if name.startswith("/") else f"/{name}"
+        add_prefix = adf_only and not slash_name.startswith(f"{PARAMETER_PREFIX}/")
+        param_prefix = PARAMETER_PREFIX if add_prefix else ""
         return f"{param_prefix}{slash_name}"
 
     def fetch_parameter(self, name, with_decryption=False, adf_only=True):
-        """Gets a Parameter from Parameter Store (Returns the Value)
-        """
+        """Gets a Parameter from Parameter Store (Returns the Value)"""
         param_name = ParameterStore._build_param_name(name, adf_only)
+        if self.cache.exists(param_name):
+            LOGGER.debug("Reading Parameter from Cache: %s", param_name)
+            cached_value = self.cache.get(param_name)
+            if isinstance(cached_value, ParameterNotFoundError):
+                raise cached_value
+            return cached_value
         try:
-            LOGGER.debug('Fetching Parameter %s', param_name)
+            LOGGER.debug("Fetching Parameter %s", param_name)
             response = self.client.get_parameter(
-                Name=param_name,
-                WithDecryption=with_decryption
+                Name=param_name, WithDecryption=with_decryption
             )
-            return response['Parameter']['Value']
+            fetched_value = response["Parameter"]["Value"]
+            self.cache.add(param_name, fetched_value)
+            return fetched_value
         except self.client.exceptions.ParameterNotFound as error:
-            LOGGER.debug('Parameter %s not found', param_name)
-            raise ParameterNotFoundError(
-                f'Parameter {param_name} Not Found',
-            ) from error
+            LOGGER.debug("Parameter %s not found", param_name)
+            not_found = ParameterNotFoundError(
+                f"Parameter {param_name} Not Found",
+            )
+            self.cache.add(param_name, not_found)
+            raise not_found from error
 
     def fetch_parameter_accept_not_found(
         self,
@@ -131,5 +137,5 @@ def fetch_parameter_accept_not_found(
         try:
             return self.fetch_parameter(name, with_decryption, adf_only)
         except ParameterNotFoundError:
-            LOGGER.debug('Using default instead: %s', default_value)
+            LOGGER.debug("Using default instead: %s", default_value)
             return default_value

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py

@@ -156,7 +156,6 @@ def __init__(
             ).lower() == "enabled"
         )
 
-
     @staticmethod
     def _account_is_active(account):
         return bool(account.get("Status") == "ACTIVE")

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_cache.py

@@ -32,3 +32,52 @@ def test_get(cls):
     assert cls.get('my_key') == 'my_value'
     assert cls.get('true_key') is True
     assert cls.get('false_key') is False
+
+
+def test_remove_existing_key(cls):
+    # Arrange
+    cls.add("test_key", "test_value")
+
+    # Act
+    cls.remove("test_key")
+
+    # Assert
+    assert cls.exists("test_key") is False
+    assert cls.get("test_key") is None
+
+
+def test_remove_non_existing_key(cls):
+    # Arrange
+    cls.remove("non_existing_key")
+
+    # Assert
+    assert cls.exists("non_existing_key") is False
+    assert cls.get("non_existing_key") is None
+
+
+def test_remove_and_read(cls):
+    # Arrange
+    cls.add("test_key", "test_value")
+
+    # Act
+    cls.remove("test_key")
+    cls.add("test_key", "new_value")
+
+    # Assert
+    assert cls.exists("test_key") is True
+    assert cls.get("test_key") == "new_value"
+
+
+def test_remove_multiple_keys(cls):
+    # Arrange
+    cls.add("key1", "value1")
+    cls.add("key2", "value2")
+
+    # Act
+    cls.remove("key1")
+
+    # Assert
+    assert cls.exists("key1") is False
+    assert cls.exists("key2") is True
+    assert cls.get("key1") is None
+    assert cls.get("key2") == "value2"