Merge pull request #113 from grahamc/rds-subnet-groups

RDS: Support VPC

Author: grahamc

nixops_aws/ec2_utils.py

@@ -12,8 +12,10 @@
 from boto.exception import BotoServerError
 from botocore.exceptions import ClientError
 from boto.pyami.config import Config
-import botocore
-from typing import Tuple
+from typing import Tuple, TYPE_CHECKING, Iterable, Any, Optional
+
+if TYPE_CHECKING:
+    import mypy_boto3_rds
 
 
 def fetch_aws_secret_key(access_key_id) -> Tuple[str, str]:
@@ -119,16 +121,33 @@ def connect_vpc(region, access_key_id):
     return conn
 
 
+def connect_rds_boto3(region, access_key_id) -> "mypy_boto3_rds.RDSClient":
+    assert region
+    (access_key_id, secret_access_key) = fetch_aws_secret_key(access_key_id)
+    client = boto3.session.Session().client(
+        "rds",
+        region_name=region,
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+    )
+    return client
+
+
 def get_access_key_id():
     return os.environ.get("EC2_ACCESS_KEY") or os.environ.get("AWS_ACCESS_KEY_ID")
 
 
-def retry(f, error_codes=[], logger=None):
+def retry(
+    f, error_codes: Optional[Iterable[Any]] = None, logger=None, num_retries: int = 7
+):
     """
         Retry function f up to 7 times. If error_codes argument is empty list, retry on all EC2 response errors,
         otherwise, only on the specified error codes.
     """
 
+    if error_codes is None:
+        error_codes = []
+
     def handle_exception(e):
         if hasattr(e, "error_code"):
             err_code = e.error_code
@@ -137,8 +156,12 @@ def handle_exception(e):
             err_code = e.response["Error"]["Code"]
             err_msg = e.response["Error"]["Message"]
 
-        if i == num_retries or (error_codes != [] and err_code not in error_codes):
-            raise e
+        if err_code == "RequestLimitExceeded":
+            return False
+
+        if error_codes and err_code not in error_codes:
+            return True
+
         if logger is not None:
             logger.log(
                 "got (possibly transient) EC2 error code '{0}': {1}. retrying...".format(
@@ -147,8 +170,9 @@ def handle_exception(e):
             )
 
     def handle_boto3_exception(e):
-        if i == num_retries:
-            raise e
+        if error_codes and getattr(e, "response", {}).get("code") not in error_codes:
+            return True
+
         if logger is not None:
             if hasattr(e, "response"):
                 logger.log(
@@ -157,29 +181,23 @@ def handle_boto3_exception(e):
                     )
                 )
 
+    def should_abort(e):
+        if isinstance(e, (SQSError, EC2ResponseError, BotoServerError)):
+            return handle_exception(e)
+        elif isinstance(e, ClientError):
+            return handle_boto3_exception(e)
+
     i = 0
-    num_retries = 7
     while i <= num_retries:
         i += 1
         next_sleep = 5 + random.random() * (2 ** i)
 
         try:
             return f()
-        except EC2ResponseError as e:
-            handle_exception(e)
-        except SQSError as e:
-            handle_exception(e)
-        except ClientError as e:
-            handle_boto3_exception(e)
-        except BotoServerError as e:
-            if e.error_code == "RequestLimitExceeded":
-                num_retries += 1
-            else:
-                handle_exception(e)
-        except botocore.exceptions.ClientError as e:
-            handle_exception(e)
         except Exception as e:
-            raise e
+            if num_retries == i or should_abort(e):
+                raise e
+            num_retries += 1
 
         time.sleep(next_sleep)
 

nixops_aws/nix/cloudwatch-log-group.nix

@@ -14,6 +14,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/cloudwatch-log-stream.nix

@@ -14,6 +14,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/default.nix

@@ -18,6 +18,7 @@
     ebsVolumes = evalResources ./ebs-volume.nix (zipAttrs resourcesByType.ebsVolumes or []);
     elasticIPs = evalResources ./elastic-ip.nix (zipAttrs resourcesByType.elasticIPs or []);
     rdsDbInstances = evalResources ./ec2-rds-dbinstance.nix (zipAttrs resourcesByType.rdsDbInstances or []);
+    rdsSubnetGroups = evalResources ./rds-subnet-group.nix (zipAttrs resourcesByType.rdsSubnetGroups or []);
     rdsDbSecurityGroups = evalResources ./ec2-rds-dbsecurity-group.nix (zipAttrs resourcesByType.rdsDbSecurityGroups or []);
     route53RecordSets = evalResources ./route53-recordset.nix (zipAttrs resourcesByType.route53RecordSets or []);
     elasticFileSystems = evalResources ./elastic-file-system.nix (zipAttrs resourcesByType.elasticFileSystems or []);

nixops_aws/nix/ec2-rds-dbinstance.nix

@@ -81,6 +81,15 @@ with import ./lib.nix lib;
       description = "The endpoint address of the database instance.  This is set by NixOps.";
     };
 
+    subnetGroup = mkOption {
+      default = null;
+      type = types.nullOr (types.either types.str (resource "rds-subnet-group"));
+      apply = x: if builtins.isNull x then null else if builtins.isString x then x else "res-" + x._name;
+      description = ''
+        RDS Subnet Group to place this database in.
+      '';
+    };
+
     securityGroups = mkOption {
       default = [ "default" ];
       type = types.listOf (types.either types.str (resource "ec2-rds-security-group"));
@@ -90,6 +99,15 @@ with import ./lib.nix lib;
       '';
     };
 
+    vpcSecurityGroups = mkOption {
+      default = null;
+      type = types.nullOr (types.listOf (types.either types.str (resource "ec2-security-group")));
+      apply = map (x: if builtins.isString x then x else "res-" + x._name);
+      description = ''
+        List of VPC security groups to authorize on this DBInstance.
+      '';
+    };
+
   };
 
   config._type = "ec2-rds-dbinstance";

nixops_aws/nix/elastic-file-system-mount-target.nix

@@ -14,6 +14,7 @@ with import ./lib.nix lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/elastic-file-system.nix

@@ -13,6 +13,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/iam-role.nix

@@ -14,6 +14,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/rds-subnet-group.nix

@@ -0,0 +1,49 @@
+{ config, lib, uuid, name, ... }:
+
+with import ./lib.nix lib;
+with lib;
+with import ./lib.nix lib;
+
+{
+
+  options = {
+
+    name = mkOption {
+      default = "nixops-${uuid}-${name}";
+      type = types.str;
+      description = "Name of the RDS subnet group.";
+    };
+
+    description = mkOption {
+      default = "NixOps-provisioned subnet group ${name}";
+      type = types.str;
+      description = "Informational description of the subnet group.";
+    };
+
+    accessKeyId = mkOption {
+      default = "";
+      type = types.str;
+      description = "The AWS Access Key ID.";
+    };
+
+    region = mkOption {
+      example = "us-east-1";
+      type = types.str;
+      description = ''
+        AWS region in which the instance is to be deployed.
+      '';
+    };
+
+    subnetIds = mkOption {
+      default = [];
+      example = [ "subnet-00000000" ];
+      type = types.listOf (types.either types.str (resource "vpc-subnet"));
+      apply = map (x: if builtins.isString x then x else "res-" + x._name + "." + x._type);
+      description = ''
+        The subnets inside a VPC to launch the databases in.
+      '';
+    };
+  };
+
+  config._type = "rds-subnet-group";
+}

nixops_aws/nix/s3-bucket.nix

@@ -19,6 +19,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/sns-topic.nix

@@ -19,6 +19,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };
 

nixops_aws/nix/sqs-queue.nix

@@ -19,6 +19,7 @@ with lib;
 
     accessKeyId = mkOption {
       type = types.str;
+      default = "";
       description = "The AWS Access Key ID.";
     };