Merge pull request #108 from grahamc/defntype

Fixup config / creation issues to work a bit more reliably

Author: grahamc

nixops_aws/backends/ec2.py

@@ -103,6 +103,8 @@ def host_key_type(self):
 class EC2State(MachineState[EC2Definition], EC2CommonState):
     """State of an EC2 machine."""
 
+    definition_type = EC2Definition
+
     @classmethod
     def get_type(cls):
         return "ec2"

nixops_aws/resources/aws_vpn_connection.py

@@ -32,6 +32,8 @@ def show_type(self):
 class AWSVPNConnectionState(nixops.resources.DiffEngineResourceState, EC2CommonState):
     """State of a AWS VPN gateway."""
 
+    definition_type = AWSVPNConnectionDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.DiffEngineResourceState.MISSING, int
     )
@@ -82,7 +84,7 @@ def create_after(self, resources, defn):
         }
 
     def realize_create_vpn_conn(self, allow_recreate):
-        config = self.get_defn()
+        config: AWSVPNConnectionDefinition = self.get_defn()
 
         if self.state == self.UP:
             if not allow_recreate:
@@ -95,8 +97,8 @@ def realize_create_vpn_conn(self, allow_recreate):
             self.warn("vpn connection definition changed, recreating ...")
             self._destroy()
 
-        self._state["region"] = config["region"]
-        customer_gtw_id = config["customerGatewayId"]
+        self._state["region"] = config.config.region
+        customer_gtw_id = config.config.customerGatewayId
         if customer_gtw_id.startswith("res-"):
             res_vpc_customer_gw = self.depl.get_typed_resource(
                 customer_gtw_id[4:].split(".")[0],
@@ -105,7 +107,7 @@ def realize_create_vpn_conn(self, allow_recreate):
             )
             customer_gtw_id = res_vpc_customer_gw._state["customerGatewayId"]
 
-        vpn_gateway_id = config["vpnGatewayId"]
+        vpn_gateway_id = config.config.vpnGatewayId
         if vpn_gateway_id.startswith("res-"):
             res_vpn_gateway = self.depl.get_typed_resource(
                 vpn_gateway_id[4:].split(".")[0], "aws-vpn-gateway", AWSVPNGatewayState
@@ -121,7 +123,7 @@ def realize_create_vpn_conn(self, allow_recreate):
             CustomerGatewayId=customer_gtw_id,
             VpnGatewayId=vpn_gateway_id,
             Type="ipsec.1",
-            Options={"StaticRoutesOnly": config["staticRoutesOnly"]},
+            Options={"StaticRoutesOnly": config.config.staticRoutesOnly},
         )
 
         vpn_conn_id = vpn_connection["VpnConnection"]["VpnConnectionId"]
@@ -130,11 +132,11 @@ def realize_create_vpn_conn(self, allow_recreate):
             self._state["vpnConnectionId"] = vpn_conn_id
             self._state["vpnGatewayId"] = vpn_gateway_id
             self._state["customerGatewayId"] = customer_gtw_id
-            self._state["staticRoutesOnly"] = config["staticRoutesOnly"]
+            self._state["staticRoutesOnly"] = config.config.staticRoutesOnly
 
     def realize_update_tag(self, allow_recreate):
-        config = self.get_defn()
-        tags = config["tags"]
+        config: AWSVPNConnectionDefinition = self.get_defn()
+        tags = {k: v for k, v in config.config.tags.items()}
         tags.update(self.get_common_tags())
         self.get_client().create_tags(
             Resources=[self._state["vpnConnectionId"]],

nixops_aws/resources/aws_vpn_connection_route.py

@@ -34,8 +34,9 @@ def show_type(self):
 class AWSVPNConnectionRouteState(
     nixops.resources.DiffEngineResourceState, EC2CommonState
 ):
-    """State of a VPN connection route"""
+    definition_type = AWSVPNConnectionRouteDefinition
 
+    """State of a VPN connection route"""
     state = nixops.util.attr_property(
         "state", nixops.resources.DiffEngineResourceState.MISSING, int
     )
@@ -79,7 +80,7 @@ def create_after(self, resources, defn):
         }
 
     def realize_create_vpn_route(self, allow_recreate):
-        config = self.get_defn()
+        config: AWSVPNConnectionRouteDefinition = self.get_defn()
 
         if self.state == self.UP:
             if not allow_recreate:
@@ -92,8 +93,8 @@ def realize_create_vpn_route(self, allow_recreate):
             self.warn("route definition changed, recreating ...")
             self._destroy()
 
-        self._state["region"] = config["region"]
-        vpn_conn_id = config["vpnConnectionId"]
+        self._state["region"] = config.config.region
+        vpn_conn_id = config.config.vpnConnectionId
         if vpn_conn_id.startswith("res-"):
             res = self.depl.get_typed_resource(
                 vpn_conn_id[4:].split(".")[0],
@@ -104,18 +105,18 @@ def realize_create_vpn_route(self, allow_recreate):
 
         self.log(
             "creating route to {0} using vpn connection {1}".format(
-                config["destinationCidrBlock"], vpn_conn_id
+                config.config.destinationCidrBlock, vpn_conn_id
             )
         )
         self.get_client().create_vpn_connection_route(
-            DestinationCidrBlock=config["destinationCidrBlock"],
+            DestinationCidrBlock=config.config.destinationCidrBlock,
             VpnConnectionId=vpn_conn_id,
         )
 
         with self.depl._db:
             self.state = self.UP
             self._state["vpnConnectionId"] = vpn_conn_id
-            self._state["destinationCidrBlock"] = config["destinationCidrBlock"]
+            self._state["destinationCidrBlock"] = config.config.destinationCidrBlock
 
     def _destroy(self):
         if self.state != self.UP:

nixops_aws/resources/aws_vpn_gateway.py

@@ -32,6 +32,8 @@ def show_type(self):
 class AWSVPNGatewayState(nixops.resources.DiffEngineResourceState, EC2CommonState):
     """State of a AWS VPN gateway."""
 
+    definition_type = AWSVPNGatewayDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.DiffEngineResourceState.MISSING, int
     )
@@ -74,7 +76,7 @@ def create_after(self, resources, defn):
         }
 
     def realize_create_vpn_gtw(self, allow_recreate):
-        config = self.get_defn()
+        config: AWSVPNGatewayDefinition = self.get_defn()
 
         if self.state == self.UP:
             if not allow_recreate:
@@ -87,17 +89,17 @@ def realize_create_vpn_gtw(self, allow_recreate):
             self.warn("VPN gateway changed, recreating...")
             self._destroy()
 
-        self._state["region"] = config["region"]
-        vpc_id = config["vpcId"]
+        self._state["region"] = config.config.region
+        vpc_id = config.config.vpcId
         if vpc_id.startswith("res-"):
             res = self.depl.get_typed_resource(
                 vpc_id[4:].split(".")[0], "vpc", VPCState
             )
             vpc_id = res._state["vpcId"]
 
-        self.log("creating VPN gateway in zone {}".format(config["zone"]))
+        self.log("creating VPN gateway in zone {}".format(config.config.zone))
         response = self.get_client().create_vpn_gateway(
-            AvailabilityZone=config["zone"], Type="ipsec.1"
+            AvailabilityZone=config.config.zone, Type="ipsec.1"
         )
 
         vpn_gtw_id = response["VpnGateway"]["VpnGatewayId"]
@@ -109,11 +111,11 @@ def realize_create_vpn_gtw(self, allow_recreate):
             self.state = self.UP
             self._state["vpnGatewayId"] = vpn_gtw_id
             self._state["vpcId"] = vpc_id
-            self._state["zone"] = config["zone"]
+            self._state["zone"] = config.config.zone
 
     def realize_update_tag(self, allow_recreate):
-        config = self.get_defn()
-        tags = config["tags"]
+        config: AWSVPNGatewayDefinition = self.get_defn()
+        tags = {k: v for k, v in config.config.tags.items()}
         tags.update(self.get_common_tags())
         self.get_client().create_tags(
             Resources=[self._state["vpnGatewayId"]],

nixops_aws/resources/cloudwatch_log_group.py

@@ -33,6 +33,8 @@ class CloudWatchLogGroupState(
 ):
     """State of the cloudwatch log group"""
 
+    definition_type = CloudWatchLogGroupDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.ResourceState.MISSING, int
     )

nixops_aws/resources/cloudwatch_log_stream.py

@@ -34,6 +34,8 @@ class CloudWatchLogStreamState(
 ):
     """State of the cloudwatch log group"""
 
+    definition_type = CloudWatchLogStreamDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.ResourceState.MISSING, int
     )

nixops_aws/resources/cloudwatch_metric_alarm.py

@@ -53,6 +53,8 @@ class CloudwatchMetricAlarmState(
 ):
     """State of a Cloudwatch Metric Alarm."""
 
+    definition_type = CloudwatchMetricAlarmDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.ResourceState.MISSING, int
     )

nixops_aws/resources/ebs_volume.py

@@ -32,6 +32,8 @@ def show_type(self):
 class EBSVolumeState(nixops.resources.ResourceState, ec2_common.EC2CommonState):
     """State of an EBS volume."""
 
+    definition_type = EBSVolumeDefinition
+
     state = nixops.util.attr_property(
         "state", nixops.resources.ResourceState.MISSING, int
     )
@@ -79,114 +81,118 @@ def _connect_boto3(self, region):
         )
         return self._conn_boto3
 
-    def _get_vol(self, config):
+    def _get_vol(self, config: EbsVolumeOptions):
         try:
-            _vol = self._connect_boto3(config["region"]).describe_volumes(
-                VolumeIds=[config["volumeId"]]
+            _vol = self._connect_boto3(config.region).describe_volumes(
+                VolumeIds=[config.volumeId]
             )["Volumes"][0]
         except botocore.exceptions.ClientError as error:
             raise error
         if _vol["VolumeType"] == "io1":
             iops = _vol["Iops"]
         else:
-            iops = config["iops"]
+            iops = config.iops
         with self.depl._db:
             self.state = self.STARTING
-            self.region = config["region"]
+            self.region = config.region
             self.zone = _vol["AvailabilityZone"]
             self.size = _vol["Size"]
-            self.volume_id = config["volumeId"]
+            self.volume_id = config.volumeId
             self.iops = iops
             self.volume_type = _vol["VolumeType"]
 
-    def create(self, defn, check, allow_reboot, allow_recreate):
+    def create(self, defn: EBSVolumeDefinition, check, allow_reboot, allow_recreate):
 
         self.access_key_id = (
-            defn.config["accessKeyId"] or nixops_aws.ec2_utils.get_access_key_id()
+            defn.config.accessKeyId or nixops_aws.ec2_utils.get_access_key_id()
         )
         if not self.access_key_id:
             raise Exception(
                 "please set ‘accessKeyId’, $EC2_ACCESS_KEY or $AWS_ACCESS_KEY_ID"
             )
 
         if self._exists():
-            if self.region != defn.config["region"] or self.zone != defn.config["zone"]:
+            if self.region != defn.config.region or self.zone != defn.config.zone:
                 raise Exception(
                     "changing the region or availability zone of an EBS volume is not supported"
                 )
 
-            if defn.config["size"] != 0 and self.size != defn.config["size"]:
+            if defn.config.size != 0 and self.size != defn.config.size:
                 raise Exception(
                     "changing the size an EBS volume is currently not supported"
                 )
 
             if (
                 self.volume_type is not None
-                and defn.config["volumeType"] != self.volume_type
+                and defn.config.volumeType != self.volume_type
             ):
                 raise Exception(
                     "changing the type of an EBS volume is currently not supported"
                 )
 
-            if defn.config["iops"] != self.iops:
+            if defn.config.iops != self.iops:
                 raise Exception(
                     "changing the IOPS of an EBS volume is currently not supported"
                 )
 
         if self.state == self.MISSING:
-            if defn.config["volumeId"]:
+            if defn.config.volumeId:
                 self.log(
-                    "Using provided EBS volume ‘{0}’...".format(defn.config["volumeId"])
+                    "Using provided EBS volume ‘{0}’...".format(defn.config.volumeId)
                 )
                 self._get_vol(defn.config)
             else:
-                if defn.config["size"] == 0 and defn.config["snapshot"] != "":
-                    snapshots = self._connect(defn.config["region"]).get_all_snapshots(
-                        snapshot_ids=[defn.config["snapshot"]]
+                if defn.config.size == 0 and defn.config.snapshot != "":
+                    snapshots = self._connect(defn.config.region).get_all_snapshots(
+                        snapshot_ids=[defn.config.snapshot]
                     )
                     assert len(snapshots) == 1
-                    defn.config["size"] = snapshots[0].volume_size
+                    defn.config.size = snapshots[0].volume_size
 
-                if defn.config["snapshot"]:
+                if defn.config.snapshot:
                     self.log(
                         "creating EBS volume of {0} GiB from snapshot ‘{1}’...".format(
-                            defn.config["size"], defn.config["snapshot"]
+                            defn.config.size, defn.config.snapshot
                         )
                     )
                 else:
                     self.log(
-                        "creating EBS volume of {0} GiB...".format(defn.config["size"])
+                        "creating EBS volume of {0} GiB...".format(defn.config.size)
                     )
 
-                if defn.config["zone"] is None:
+                if defn.config.zone is None:
                     raise Exception(
                         "please set a zone where the volume will be created"
                     )
 
-                volume = self._connect(defn.config["region"]).create_volume(
-                    zone=defn.config["zone"],
-                    size=defn.config["size"],
-                    snapshot=defn.config["snapshot"],
-                    iops=defn.config["iops"],
-                    volume_type=defn.config["volumeType"],
+                volume = self._connect(defn.config.region).create_volume(
+                    zone=defn.config.zone,
+                    size=defn.config.size,
+                    snapshot=defn.config.snapshot,
+                    iops=defn.config.iops,
+                    volume_type=defn.config.volumeType,
                 )
                 # FIXME: if we crash before the next step, we forget the
                 # volume we just created.  Doesn't seem to be anything we
                 # can do about this.
 
                 with self.depl._db:
                     self.state = self.STARTING
-                    self.region = defn.config["region"]
-                    self.zone = defn.config["zone"]
-                    self.size = defn.config["size"]
+                    self.region = defn.config.region
+                    self.zone = defn.config.zone
+                    self.size = defn.config.size
                     self.volume_id = volume.id
-                    self.iops = defn.config["iops"]
-                    self.volume_type = defn.config["volumeType"]
+                    self.iops = defn.config.iops
+                    self.volume_type = defn.config.volumeType
 
                 self.log("volume ID is ‘{0}’".format(self.volume_id))
 
         if self.state == self.STARTING or check:
-            self.update_tags(self.volume_id, user_tags=defn.config["tags"], check=check)
+            # ensure the connection has been established before calling
+            # update_tags
+            self._connect(self.region)
+
+            self.update_tags(self.volume_id, user_tags=defn.config.tags, check=check)
             nixops_aws.ec2_utils.wait_for_volume_available(
                 self._connect(self.region),
                 self.volume_id,

nixops_aws/resources/ec2_common.py

@@ -10,7 +10,7 @@
 from nixops.state import StateDict
 from typing import Optional
 from boto.ec2.connection import EC2Connection
-from typing import TYPE_CHECKING
+from typing import Mapping, TYPE_CHECKING
 
 if TYPE_CHECKING:
     import mypy_boto3_ec2
@@ -33,7 +33,7 @@ def _retry(self, fun, **kwargs):
 
     tags = nixops.util.attr_property("ec2.tags", {}, "json")
 
-    def get_common_tags(self):
+    def get_common_tags(self) -> Mapping[str, str]:
         tags = {
             "CharonNetworkUUID": self.depl.uuid,
             "CharonMachineName": self.name,
@@ -75,7 +75,7 @@ def get_client(self):
         # This class is weird and doesn't have it's full dependencies declared.
         # This function will _only_ work when _also_ inheriting from DiffEngineResourceState
         new_access_key_id = (
-            self.get_defn()["accessKeyId"] if self.depl.definitions else None  # type: ignore
+            self.get_defn().config.accessKeyId if self.depl.definitions else None  # type: ignore
         ) or nixops_aws.ec2_utils.get_access_key_id()
         if new_access_key_id is not None:
             self.access_key_id = new_access_key_id