From 73dc506bf5a50966a63f07bf1a97a8c057c77284 Mon Sep 17 00:00:00 2001 From: Rafael Pereyra Date: Fri, 24 Oct 2025 09:01:04 -0400 Subject: [PATCH 1/3] refactor: improve CDK infrastructure maintainability and documentation - Remove hardcoded CloudWatch log group names to prevent deployment conflicts - Add descriptive CloudFormation output descriptions for better resource identification - Remove unused Names import and logGroupName parameters - Add CDK-nag rule to enforce dynamic log group naming - Set default values for CloudFormation template parameters --- src/cdk/lib/constructs/cloudtrail.ts | 4 +--- src/cdk/lib/constructs/database.ts | 4 ++++ src/cdk/lib/constructs/dynamodb.ts | 6 +++++ src/cdk/lib/constructs/ecs-service.ts | 7 ++---- src/cdk/lib/constructs/ecs.ts | 3 +++ src/cdk/lib/constructs/eks.ts | 7 ++++++ src/cdk/lib/constructs/eventbus.ts | 2 ++ src/cdk/lib/constructs/microservice.ts | 1 - src/cdk/lib/constructs/network.ts | 22 +++++++++++++++-- .../lib/constructs/opensearch-application.ts | 2 ++ .../lib/constructs/opensearch-collection.ts | 4 ++++ src/cdk/lib/constructs/opensearch-pipeline.ts | 4 +++- src/cdk/lib/constructs/queue.ts | 3 +++ src/cdk/lib/constructs/vpc-endpoints.ts | 12 ++++++++++ src/cdk/lib/constructs/waf.ts | 4 +--- src/cdk/lib/microservices/petfood-agent.ts | 2 +- src/cdk/lib/microservices/petsite.ts | 1 + src/cdk/lib/pipeline.ts | 1 + .../status-updater/status-updater.ts | 3 +-- src/cdk/lib/utils/workshop-nag-pack.ts | 24 +++++++++++++++++++ .../codebuild-deployment-template.yaml | 2 ++ 21 files changed, 100 insertions(+), 18 deletions(-) diff --git a/src/cdk/lib/constructs/cloudtrail.ts b/src/cdk/lib/constructs/cloudtrail.ts index 19dee19e..8d4b22a3 100644 --- a/src/cdk/lib/constructs/cloudtrail.ts +++ b/src/cdk/lib/constructs/cloudtrail.ts @@ -16,7 +16,7 @@ import { Construct } from 'constructs'; import { Trail, InsightType, CfnEventDataStore, CfnTrail } from 'aws-cdk-lib/aws-cloudtrail'; import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { Role, ServicePrincipal, PolicyStatement, PolicyDocument } from 'aws-cdk-lib/aws-iam'; -import { Names, RemovalPolicy, Duration } from 'aws-cdk-lib'; +import { RemovalPolicy, Duration } from 'aws-cdk-lib'; import { NagSuppressions } from 'cdk-nag'; import { BlockPublicAccess, Bucket } from 'aws-cdk-lib/aws-s3'; @@ -58,11 +58,9 @@ export class WorkshopCloudTrail extends Construct { constructor(scope: Construct, id: string, properties: WorkshopCloudTrailProperties) { super(scope, id); - const logName = Names.uniqueResourceName(this, {}); // Create CloudWatch log group for CloudTrail this.logGroup = new LogGroup(this, 'CloudTrailLogGroup', { retention: properties.logRetentionDays || RetentionDays.ONE_WEEK, - logGroupName: `/aws/cloudtrail/${logName}`, removalPolicy: RemovalPolicy.DESTROY, }); diff --git a/src/cdk/lib/constructs/database.ts b/src/cdk/lib/constructs/database.ts index 5e205b36..8db9539a 100644 --- a/src/cdk/lib/constructs/database.ts +++ b/src/cdk/lib/constructs/database.ts @@ -158,21 +158,25 @@ export class AuroraDatabase extends Construct { new CfnOutput(this, 'ClusterArn', { value: this.cluster.clusterArn, exportName: AURORA_CLUSTER_ARN_EXPORT_NAME, + description: 'ARN of the Aurora PostgreSQL database cluster', }); new CfnOutput(this, 'ClusterEndpoint', { value: this.cluster.clusterEndpoint.hostname, exportName: AURORA_CLUSTER_ENDPOINT_EXPORT_NAME, + description: 'Writer endpoint hostname for the Aurora PostgreSQL cluster', }); new CfnOutput(this, 'SecurityGroupId', { value: this.databaseSecurityGroup.securityGroupId, exportName: AURORA_SECURITY_GROUP_ID_EXPORT_NAME, + description: 'Security group ID for Aurora PostgreSQL cluster access control', }); new CfnOutput(this, 'AdminSecretArn', { value: this.cluster.secret!.secretArn, exportName: AURORA_ADMIN_SECRET_ARN_EXPORT_NAME, + description: 'Secrets Manager ARN containing Aurora admin credentials', }); } diff --git a/src/cdk/lib/constructs/dynamodb.ts b/src/cdk/lib/constructs/dynamodb.ts index 273bef2d..399b7005 100644 --- a/src/cdk/lib/constructs/dynamodb.ts +++ b/src/cdk/lib/constructs/dynamodb.ts @@ -147,31 +147,37 @@ export class DynamoDatabase extends Construct { new CfnOutput(this, 'TableArn', { value: this.petAdoptionTable.tableArn, exportName: DYNAMODB_TABLE_ARN_EXPORT_NAME, + description: 'ARN of the DynamoDB table for pet adoption data', }); new CfnOutput(this, 'TableName', { value: this.petAdoptionTable.tableName, exportName: DYNAMODB_TABLE_NAME_EXPORT_NAME, + description: 'Name of the DynamoDB table for pet adoption data', }); new CfnOutput(this, 'PetFoodsTableArn', { value: this.petFoodsTable.tableArn, exportName: `${DYNAMODB_TABLE_ARN_EXPORT_NAME}-PetFoods`, + description: 'ARN of the DynamoDB table for pet food products', }); new CfnOutput(this, 'PetFoodsTableName', { value: this.petFoodsTable.tableName, exportName: `${DYNAMODB_TABLE_NAME_EXPORT_NAME}-PetFoods`, + description: 'Name of the DynamoDB table for pet food products', }); new CfnOutput(this, 'PetFoodsCartTableArn', { value: this.petFoodsCartTable.tableArn, exportName: `${DYNAMODB_TABLE_ARN_EXPORT_NAME}-PetFoodsCart`, + description: 'ARN of the DynamoDB table for pet food shopping cart', }); new CfnOutput(this, 'PetFoodsCartTableName', { value: this.petFoodsCartTable.tableName, exportName: `${DYNAMODB_TABLE_NAME_EXPORT_NAME}-PetFoodsCart`, + description: 'Name of the DynamoDB table for pet food shopping cart', }); } diff --git a/src/cdk/lib/constructs/ecs-service.ts b/src/cdk/lib/constructs/ecs-service.ts index b520ae71..0942edc6 100644 --- a/src/cdk/lib/constructs/ecs-service.ts +++ b/src/cdk/lib/constructs/ecs-service.ts @@ -103,7 +103,6 @@ export abstract class EcsService extends Microservice { // Create CloudWatch log group const logGroup = new LogGroup(this, 'ecs-log-group', { - logGroupName: properties.logGroupName || `/ecs/${properties.name}`, removalPolicy: RemovalPolicy.DESTROY, retention: properties.logRetentionDays || RetentionDays.ONE_WEEK, }); @@ -210,7 +209,7 @@ export abstract class EcsService extends Microservice { this.addAdotPythonInitContainer(taskDefinition, container); // Add CloudWatch agent sidecar - this.addCloudWatchAgentSidecar(taskDefinition, properties); + this.addCloudWatchAgentSidecar(taskDefinition); } if (!properties.disableService) { @@ -474,7 +473,6 @@ export abstract class EcsService extends Microservice { logging: new AwsLogDriver({ streamPrefix: 'firelens', logGroup: new LogGroup(this, 'firelens-log-group', { - logGroupName: `/ecs/firelens/${properties.name}`, removalPolicy: RemovalPolicy.DESTROY, retention: RetentionDays.ONE_WEEK, }), @@ -568,7 +566,7 @@ export abstract class EcsService extends Microservice { }); } - private addCloudWatchAgentSidecar(taskDefinition: TaskDefinition, properties: EcsServiceProperties): void { + private addCloudWatchAgentSidecar(taskDefinition: TaskDefinition): void { // CloudWatch agent configuration for Application Signals const cloudWatchConfig = { traces: { @@ -592,7 +590,6 @@ export abstract class EcsService extends Microservice { logging: new AwsLogDriver({ streamPrefix: 'cloudwatch-agent', logGroup: new LogGroup(this, 'cloudwatch-agent-log-group', { - logGroupName: `/ecs/cloudwatch-agent/${properties.name}`, removalPolicy: RemovalPolicy.DESTROY, retention: RetentionDays.ONE_WEEK, }), diff --git a/src/cdk/lib/constructs/ecs.ts b/src/cdk/lib/constructs/ecs.ts index df7902dc..4b2f04d5 100644 --- a/src/cdk/lib/constructs/ecs.ts +++ b/src/cdk/lib/constructs/ecs.ts @@ -151,16 +151,19 @@ export class WorkshopEcs extends Construct { new CfnOutput(this, 'ClusterArn', { value: this.cluster.clusterArn, exportName: ECS_CLUSTER_ARN_EXPORT_NAME, + description: 'ARN of the ECS cluster for container deployments', }); new CfnOutput(this, 'ClusterName', { value: this.cluster.clusterName, exportName: ECS_CLUSTER_NAME_EXPORT_NAME, + description: 'Name of the ECS cluster', }); new CfnOutput(this, 'SecurityGroupId', { value: this.securityGroup.securityGroupId, exportName: ECS_SECURITY_GROUP_ID_EXPORT_NAME, + description: 'Security group ID for ECS cluster resources', }); } diff --git a/src/cdk/lib/constructs/eks.ts b/src/cdk/lib/constructs/eks.ts index 5297aa71..f44964c6 100644 --- a/src/cdk/lib/constructs/eks.ts +++ b/src/cdk/lib/constructs/eks.ts @@ -235,36 +235,43 @@ export class WorkshopEks extends Construct { new CfnOutput(this, 'ClusterArn', { value: this.cluster.clusterArn, exportName: EKS_CLUSTER_ARN_EXPORT_NAME, + description: 'ARN of the EKS cluster', }); new CfnOutput(this, 'ClusterName', { value: this.cluster.clusterName, exportName: EKS_CLUSTER_NAME_EXPORT_NAME, + description: 'Name of the EKS cluster for kubectl configuration', }); new CfnOutput(this, 'SecurityGroupId', { value: this.cluster.clusterSecurityGroupId, exportName: EKS_SECURITY_GROUP_ID_EXPORT_NAME, + description: 'Security group ID for EKS cluster network access', }); new CfnOutput(this, 'KubectlRoleArn', { value: this.cluster.kubectlRole!.roleArn, exportName: EKS_KUBECTL_ROLE_ARN_EXPORT_NAME, + description: 'IAM role ARN for kubectl operations on the EKS cluster', }); new CfnOutput(this, 'OpenIdConnectProviderArn', { value: this.cluster.openIdConnectProvider.openIdConnectProviderArn, exportName: EKS_OPEN_ID_CONNECT_PROVIDER_ARN_EXPORT_NAME, + description: 'OIDC provider ARN for EKS service account IAM role integration', }); new CfnOutput(this, 'KubectlSecurityGroupId', { value: this.cluster.kubectlSecurityGroup!.securityGroupId, exportName: EKS_KUBECTL_SECURITY_GROUP_ID_EXPORT_NAME, + description: 'Security group ID for kubectl Lambda function', }); new CfnOutput(this, 'KubectlLambdaRoleArn', { value: this.cluster.kubectlLambdaRole!.roleArn, exportName: EKS_KUBECTL_LAMBDA_ROLE_ARN_EXPORT_NAME, + description: 'IAM role ARN for the kubectl Lambda function', }); } diff --git a/src/cdk/lib/constructs/eventbus.ts b/src/cdk/lib/constructs/eventbus.ts index 777be909..8b633844 100644 --- a/src/cdk/lib/constructs/eventbus.ts +++ b/src/cdk/lib/constructs/eventbus.ts @@ -75,10 +75,12 @@ export class EventBusResources extends Construct { new CfnOutput(this, 'EventBusArn', { value: this.eventBus.eventBusArn, exportName: EVENTBUS_ARN_EXPORT_NAME, + description: 'ARN of the EventBridge event bus for cross-service communication', }); new CfnOutput(this, 'EventBusName', { value: this.eventBus.eventBusName, exportName: EVENTBUS_NAME_EXPORT_NAME, + description: 'Name of the EventBridge event bus', }); } } diff --git a/src/cdk/lib/constructs/microservice.ts b/src/cdk/lib/constructs/microservice.ts index 142c7eba..91490873 100644 --- a/src/cdk/lib/constructs/microservice.ts +++ b/src/cdk/lib/constructs/microservice.ts @@ -42,7 +42,6 @@ export interface MicroserviceProperties { name: string; repositoryURI: string; disableService?: boolean; - logGroupName?: string; healthCheck?: string; subnetType?: SubnetType; listenerPort?: number; diff --git a/src/cdk/lib/constructs/network.ts b/src/cdk/lib/constructs/network.ts index 6993ed35..2d750de0 100644 --- a/src/cdk/lib/constructs/network.ts +++ b/src/cdk/lib/constructs/network.ts @@ -269,35 +269,50 @@ export class WorkshopNetwork extends Construct { * Creates CloudFormation outputs for VPC resources */ private createVpcOutputs() { - new CfnOutput(this, 'VpcId', { value: this.vpc.vpcId, exportName: VPC_ID_EXPORT_NAME }); - new CfnOutput(this, 'VpcCidr', { value: this.vpc.vpcCidrBlock, exportName: VPC_CIDR_EXPORT_NAME }); + new CfnOutput(this, 'VpcId', { + value: this.vpc.vpcId, + exportName: VPC_ID_EXPORT_NAME, + description: 'VPC ID for the workshop network', + }); + new CfnOutput(this, 'VpcCidr', { + value: this.vpc.vpcCidrBlock, + exportName: VPC_CIDR_EXPORT_NAME, + description: 'CIDR block of the workshop VPC', + }); new CfnOutput(this, 'VpcPrivateSubnets', { value: this.vpc.privateSubnets.map((s) => s.subnetId).join(','), exportName: VPC_PRIVATE_SUBNETS_EXPORT_NAME, + description: 'Comma-separated list of private subnet IDs with NAT gateway access', }); new CfnOutput(this, 'VpcPublicSubnets', { value: this.vpc.publicSubnets.map((s) => s.subnetId).join(','), exportName: VPC_PUBLIC_SUBNETS_EXPORT_NAME, + description: 'Comma-separated list of public subnet IDs with internet gateway access', }); new CfnOutput(this, 'VpcIsolatedSubnets', { value: this.vpc.isolatedSubnets.map((s) => s.subnetId).join(','), exportName: VPC_ISOLATED_SUBNETS_EXPORT_NAME, + description: 'Comma-separated list of isolated subnet IDs without internet access', }); new CfnOutput(this, 'VpcAvailabilityZones', { value: this.vpc.availabilityZones.join(','), exportName: VPC_AVAILABILITY_ZONES_EXPORT_NAME, + description: 'Comma-separated list of availability zones used by the VPC', }); new CfnOutput(this, 'VpcPrivateSubnetCidrs', { value: this.vpc.privateSubnets.map((s) => s.ipv4CidrBlock).join(','), exportName: VPC_PRIVATE_SUBNET_CIDRS_EXPORT_NAME, + description: 'Comma-separated list of CIDR blocks for private subnets', }); new CfnOutput(this, 'VpcPublicSubnetCidrs', { value: this.vpc.publicSubnets.map((s) => s.ipv4CidrBlock).join(','), exportName: VPC_PUBLIC_SUBNET_CIDRS_EXPORT_NAME, + description: 'Comma-separated list of CIDR blocks for public subnets', }); new CfnOutput(this, 'VpcIsolatedSubnetCidrs', { value: this.vpc.isolatedSubnets.map((s) => s.ipv4CidrBlock).join(','), exportName: VPC_ISOLATED_SUBNET_CIDRS_EXPORT_NAME, + description: 'Comma-separated list of CIDR blocks for isolated subnets', }); } @@ -308,16 +323,19 @@ export class WorkshopNetwork extends Construct { new CfnOutput(this, 'CloudMapNamespaceId', { value: this.cloudMapNamespace.namespaceId, exportName: CLOUDMAP_NAMESPACE_ID_EXPORT_NAME, + description: 'Cloud Map namespace ID for service discovery', }); new CfnOutput(this, 'CloudMapNamespaceName', { value: this.cloudMapNamespace.namespaceName, exportName: CLOUDMAP_NAMESPACE_NAME_EXPORT_NAME, + description: 'Cloud Map namespace name for service discovery', }); new CfnOutput(this, 'CloudMapNamespaceArn', { value: this.cloudMapNamespace.namespaceArn, exportName: CLOUDMAP_NAMESPACE_ARN_EXPORT_NAME, + description: 'Cloud Map namespace ARN for service discovery', }); } diff --git a/src/cdk/lib/constructs/opensearch-application.ts b/src/cdk/lib/constructs/opensearch-application.ts index 772b0e59..e790e3d9 100644 --- a/src/cdk/lib/constructs/opensearch-application.ts +++ b/src/cdk/lib/constructs/opensearch-application.ts @@ -100,11 +100,13 @@ export class OpenSearchApplication extends Construct { new CfnOutput(this, 'ApplicationArn', { value: this.application.attrArn, exportName: OPENSEARCH_APPLICATION_ARN_EXPORT_NAME, + description: 'ARN of the OpenSearch UI application for data visualization', }); new CfnOutput(this, 'ApplicationId', { value: this.application.attrId, exportName: OPENSEARCH_APPLICATION_ID_EXPORT_NAME, + description: 'ID of the OpenSearch UI application', }); } diff --git a/src/cdk/lib/constructs/opensearch-collection.ts b/src/cdk/lib/constructs/opensearch-collection.ts index 80e209b7..d27713e9 100644 --- a/src/cdk/lib/constructs/opensearch-collection.ts +++ b/src/cdk/lib/constructs/opensearch-collection.ts @@ -177,22 +177,26 @@ export class OpenSearchCollection extends Construct { new CfnOutput(this, 'CollectionArn', { value: this.collection.attrArn, exportName: OPENSEARCH_COLLECTION_ARN_EXPORT_NAME, + description: 'ARN of the OpenSearch Serverless collection for log storage', }); new CfnOutput(this, 'CollectionId', { value: this.collection.attrId, exportName: OPENSEARCH_COLLECTION_ID_EXPORT_NAME, + description: 'ID of the OpenSearch Serverless collection', }); new CfnOutput(this, 'CollectionEndpoint', { value: this.collection.attrCollectionEndpoint, exportName: OPENSEARCH_COLLECTION_ENDPOINT_EXPORT_NAME, + description: 'HTTPS endpoint URL for the OpenSearch Serverless collection', }); // Export access policy name for updates new CfnOutput(this, 'AccessPolicyName', { value: this.accessPolicy.name!, exportName: `${OPENSEARCH_COLLECTION_ARN_EXPORT_NAME}-AccessPolicy`, + description: 'Name of the data access policy for the OpenSearch collection', }); } diff --git a/src/cdk/lib/constructs/opensearch-pipeline.ts b/src/cdk/lib/constructs/opensearch-pipeline.ts index 91bf9aa2..dd8009d4 100644 --- a/src/cdk/lib/constructs/opensearch-pipeline.ts +++ b/src/cdk/lib/constructs/opensearch-pipeline.ts @@ -166,7 +166,6 @@ export class OpenSearchPipeline extends Construct { // Create CloudWatch log group for pipeline logs // OpenSearch Ingestion requires log groups to use /aws/vendedlogs/ prefix const logGroup = new LogGroup(this, 'PipelineLogGroup', { - logGroupName: `/aws/vendedlogs/opensearch-ingestion/${pipelineName}`, retention: RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, }); @@ -271,16 +270,19 @@ log-pipeline: new CfnOutput(this, 'PipelineArn', { value: this.pipeline.attrPipelineArn, exportName: OPENSEARCH_PIPELINE_ARN_EXPORT_NAME, + description: 'ARN of the OpenSearch Ingestion pipeline for log processing', }); new CfnOutput(this, 'PipelineEndpoint', { value: this.pipelineEndpoint, exportName: OPENSEARCH_PIPELINE_ENDPOINT_EXPORT_NAME, + description: 'HTTP endpoint URL for ingesting logs into the OpenSearch pipeline', }); new CfnOutput(this, 'PipelineRoleArn', { value: this.pipelineRole.roleArn, exportName: OPENSEARCH_PIPELINE_ROLE_ARN_EXPORT_NAME, + description: 'IAM role ARN used by the OpenSearch Ingestion pipeline', }); } diff --git a/src/cdk/lib/constructs/queue.ts b/src/cdk/lib/constructs/queue.ts index e565bb6a..1f25fb36 100644 --- a/src/cdk/lib/constructs/queue.ts +++ b/src/cdk/lib/constructs/queue.ts @@ -116,14 +116,17 @@ export class QueueResources extends Construct { new CfnOutput(this, 'SNSTopicArn', { value: this.topic.topicArn, exportName: SNS_TOPIC_ARN_EXPORT_NAME, + description: 'ARN of the SNS topic for pet adoption notifications', }); new CfnOutput(this, 'SQSQueueArn', { value: this.queue.queueArn, exportName: SQS_QUEUE_ARN_EXPORT_NAME, + description: 'ARN of the SQS queue for pet adoption messages', }); new CfnOutput(this, 'SQSQueueUrl', { value: this.queue.queueUrl, exportName: SQS_QUEUE_URL_EXPORT_NAME, + description: 'URL of the SQS queue for sending and receiving messages', }); } diff --git a/src/cdk/lib/constructs/vpc-endpoints.ts b/src/cdk/lib/constructs/vpc-endpoints.ts index c703b500..93b9d838 100644 --- a/src/cdk/lib/constructs/vpc-endpoints.ts +++ b/src/cdk/lib/constructs/vpc-endpoints.ts @@ -137,61 +137,73 @@ export class VpcEndpoints extends Construct { new CfnOutput(this, 'ApiGatewayEndpointId', { value: this.apiGatewayEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_APIGATEWAY_ID_EXPORT_NAME, + description: 'VPC endpoint ID for API Gateway private access', }); new CfnOutput(this, 'DynamoDbEndpointId', { value: this.dynamoDbEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_DYNAMODB_ID_EXPORT_NAME, + description: 'VPC endpoint ID for DynamoDB private access', }); new CfnOutput(this, 'LambdaEndpointId', { value: this.lambdaEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_LAMBDA_ID_EXPORT_NAME, + description: 'VPC endpoint ID for Lambda private access', }); new CfnOutput(this, 'ServiceDiscoveryEndpointId', { value: this.serviceDiscoveryEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_SERVICEDISCOVERY_ID_EXPORT_NAME, + description: 'VPC endpoint ID for Cloud Map service discovery', }); new CfnOutput(this, 'DataServiceDiscoveryEndpointId', { value: this.dataServiceDiscoveryEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_DATA_SERVICEDISCOVERY_ID_EXPORT_NAME, + description: 'VPC endpoint ID for Cloud Map data service discovery', }); new CfnOutput(this, 'S3EndpointId', { value: this.s3Endpoint.vpcEndpointId, exportName: VPC_ENDPOINT_S3_ID_EXPORT_NAME, + description: 'VPC endpoint ID for S3 private access', }); new CfnOutput(this, 'SSMEndpointId', { value: this.ssmEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_SSM_ID_EXPORT_NAME, + description: 'VPC endpoint ID for Systems Manager private access', }); new CfnOutput(this, 'EC2MessagesEndpointId', { value: this.ec2MessagesEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_EC2MESSAGES_ID_EXPORT_NAME, + description: 'VPC endpoint ID for EC2 messages (SSM Session Manager)', }); new CfnOutput(this, 'SSMMessagesEndpointId', { value: this.ssmMessagesEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_SSMMESSAGES_ID_EXPORT_NAME, + description: 'VPC endpoint ID for SSM messages (Session Manager)', }); new CfnOutput(this, 'SecretsManagerEndpointId', { value: this.secretsManagerEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_SECRETSMANAGER_ID_EXPORT_NAME, + description: 'VPC endpoint ID for Secrets Manager private access', }); new CfnOutput(this, 'CloudWatchMonitoringEndpointId', { value: this.cloudWatchMonitoringEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_CLOUDWATCH_MONITORING_ID_EXPORT_NAME, + description: 'VPC endpoint ID for CloudWatch monitoring metrics', }); new CfnOutput(this, 'CloudWatchLogsEndpointId', { value: this.cloudWatchLogsEndpoint.vpcEndpointId, exportName: VPC_ENDPOINT_CLOUDWATCH_LOGS_ID_EXPORT_NAME, + description: 'VPC endpoint ID for CloudWatch Logs', }); } diff --git a/src/cdk/lib/constructs/waf.ts b/src/cdk/lib/constructs/waf.ts index 88ac9d33..9b7941dc 100644 --- a/src/cdk/lib/constructs/waf.ts +++ b/src/cdk/lib/constructs/waf.ts @@ -1,4 +1,4 @@ -import { Names, RemovalPolicy } from 'aws-cdk-lib'; +import { RemovalPolicy } from 'aws-cdk-lib'; import { CfnLoggingConfiguration, CfnWebACL } from 'aws-cdk-lib/aws-wafv2'; import { Construct } from 'constructs'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; @@ -26,7 +26,6 @@ export class RegionalWaf extends Construct { const logGroup = new LogGroup(this, 'WAFv2RegionalLogGroup', { retention: properties.logRetention || RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, - logGroupName: 'aws-waf-logs-regional-' + Names.uniqueId(this), }); const webAcl = new CfnWebACL(this, 'WAFv2RegionalACL', { @@ -101,7 +100,6 @@ export class GlobalWaf extends Construct { const logGroup = new LogGroup(this, 'WAFv2GlobalLogGroup', { retention: properties.logRetention || RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, - logGroupName: 'aws-waf-logs-global-' + Names.uniqueId(this), }); const webAcl = new CfnWebACL(this, 'WAFv2GlobalACL', { defaultAction: { diff --git a/src/cdk/lib/microservices/petfood-agent.ts b/src/cdk/lib/microservices/petfood-agent.ts index 9c8953bd..0ac50257 100644 --- a/src/cdk/lib/microservices/petfood-agent.ts +++ b/src/cdk/lib/microservices/petfood-agent.ts @@ -189,7 +189,7 @@ export class PetFoodAgentConstruct extends Construct { new CfnOutput(this, 'AgentRuntimeArn', { value: this.agentRuntime.attrAgentRuntimeArn, - description: 'Agent Runtime ARN', + description: 'ARN of the Bedrock Agent Runtime for pet food recommendations', }); } } diff --git a/src/cdk/lib/microservices/petsite.ts b/src/cdk/lib/microservices/petsite.ts index 6bdc8d63..8af7049d 100644 --- a/src/cdk/lib/microservices/petsite.ts +++ b/src/cdk/lib/microservices/petsite.ts @@ -297,6 +297,7 @@ export class PetSite extends EKSDeployment { new CfnOutput(this, 'PetSiteUrl', { value: `https://${this.distribution.distributionDomainName}`, exportName: 'PetSiteUrl', + description: 'The URL of the PetSite application', }); if (this.loadBalancer) { diff --git a/src/cdk/lib/pipeline.ts b/src/cdk/lib/pipeline.ts index 25a792ca..c0f9a170 100644 --- a/src/cdk/lib/pipeline.ts +++ b/src/cdk/lib/pipeline.ts @@ -399,6 +399,7 @@ export class CDKPipeline extends Stack { new CfnOutput(this, 'PipelineArn', { value: pipeline.pipeline.pipelineArn, exportName: 'PipelineArn', + description: 'ARN of the CI/CD pipeline for deploying workshop infrastructure', }); /** diff --git a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts index 22162b79..d398b378 100644 --- a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts +++ b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts @@ -12,7 +12,7 @@ import { import { Construct } from 'constructs'; import { ManagedPolicy, PolicyDocument, Effect, PolicyStatement, StarPrincipal } from 'aws-cdk-lib/aws-iam'; import { ILayerVersion, LayerVersion } from 'aws-cdk-lib/aws-lambda'; -import { Names, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack } from 'aws-cdk-lib'; import { BundlingOptions } from 'aws-cdk-lib/aws-lambda-nodejs'; import { EndpointType, LambdaRestApi, LogGroupLogDestination, MethodLoggingLevel } from 'aws-cdk-lib/aws-apigateway'; import { NagSuppressions } from 'cdk-nag'; @@ -35,7 +35,6 @@ export class StatusUpdatedService extends WokshopLambdaFunction { super(scope, id, properties); const accesLogs = new LogGroup(this, 'access-logs', { - logGroupName: `/aws/apigw/${Names.uniqueId(this)}`, retention: properties.logRetentionDays || RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, }); diff --git a/src/cdk/lib/utils/workshop-nag-pack.ts b/src/cdk/lib/utils/workshop-nag-pack.ts index 8b2b9d66..639bd96b 100644 --- a/src/cdk/lib/utils/workshop-nag-pack.ts +++ b/src/cdk/lib/utils/workshop-nag-pack.ts @@ -33,6 +33,16 @@ export class WorkshopNagPack extends NagPack { node: node, }); + this.applyRule({ + ruleSuffixOverride: 'CWL3', + info: 'CloudWatch Log Groups should use dynamically generated names', + explanation: + 'Log groups with static names may cause conflicts when redeploying the stack. Use dynamically generated names instead.', + level: NagMessageLevel.ERROR, + rule: this.checkCloudWatchLogGroupName, + node: node, + }); + this.applyRule({ ruleSuffixOverride: 'S3-1', info: 'S3 Buckets should have deletion policy configured', @@ -90,6 +100,20 @@ export class WorkshopNagPack extends NagPack { return NagRuleCompliance.NOT_APPLICABLE; }; + private checkCloudWatchLogGroupName = (node: CfnResource): NagRuleResult => { + if (node.cfnResourceType === 'AWS::Logs::LogGroup') { + const logGroupName = NagRules.resolveIfPrimitive( + node, + (node as CfnResource & { logGroupName?: unknown }).logGroupName, + ); + if (logGroupName && typeof logGroupName === 'string' && !logGroupName.includes('Ref')) { + return NagRuleCompliance.NON_COMPLIANT; + } + return NagRuleCompliance.COMPLIANT; + } + return NagRuleCompliance.NOT_APPLICABLE; + }; + private checkS3BucketDeletion = (node: CfnResource): NagRuleResult => { if (node.cfnResourceType === 'AWS::S3::Bucket') { const deletionPolicy = node.cfnOptions.deletionPolicy; diff --git a/src/templates/codebuild-deployment-template.yaml b/src/templates/codebuild-deployment-template.yaml index 09a30893..634a7054 100644 --- a/src/templates/codebuild-deployment-template.yaml +++ b/src/templates/codebuild-deployment-template.yaml @@ -139,6 +139,7 @@ Parameters: ConstraintDescription: 'Must match the allowable values for a Tag Key. This can only contain alphanumeric characters or special characters ( _ . : / = + - or @) up to 128 characters' + Default: awsApplication pUserDefinedTagValue3: Type: String @@ -147,6 +148,7 @@ Parameters: ConstraintDescription: 'Must match the allowable values for a Tag Value. This can only contain alphanumeric characters or special characters ( _ . : / = + - or @) up to 256 characters' + Default: One Observabiity Workshop pUserDefinedTagKey4: Type: String From a3b69625b86a585ac3327b5d6f8bf26c6191c74e Mon Sep 17 00:00:00 2001 From: Rafael Pereyra Date: Fri, 24 Oct 2025 12:50:01 -0400 Subject: [PATCH 2/3] feat: add exports management system and enhance CDK infrastructure - Modified CDK configuration constants and local deployment settings - Enhanced WAF construct and pipeline configuration - Improved status updater function and utility functions - Refined workshop nag pack rules - Added complete exports management system with Python script, dashboard template, and documentation - Updated CodeBuild deployment template --- src/cdk/bin/constants.ts | 114 +-- src/cdk/bin/local.ts | 1 + src/cdk/lib/constructs/waf.ts | 18 +- src/cdk/lib/pipeline.ts | 31 + .../status-updater/status-updater.ts | 8 +- src/cdk/lib/utils/utilities.ts | 7 +- src/cdk/lib/utils/workshop-nag-pack.ts | 26 +- src/cdk/scripts/README.md | 194 +++++ src/cdk/scripts/manage-exports.py | 636 +++++++++++++++ src/cdk/scripts/requirements.txt | 18 + .../scripts/templates/exports-dashboard.j2 | 739 ++++++++++++++++++ src/cdk/scripts/test-dashboard.html | 440 +++++++++++ .../codebuild-deployment-template.yaml | 41 + 13 files changed, 2205 insertions(+), 68 deletions(-) create mode 100644 src/cdk/scripts/README.md create mode 100755 src/cdk/scripts/manage-exports.py create mode 100644 src/cdk/scripts/requirements.txt create mode 100644 src/cdk/scripts/templates/exports-dashboard.j2 create mode 100644 src/cdk/scripts/test-dashboard.html diff --git a/src/cdk/bin/constants.ts b/src/cdk/bin/constants.ts index db2d7fc4..f35ac296 100644 --- a/src/cdk/bin/constants.ts +++ b/src/cdk/bin/constants.ts @@ -21,89 +21,89 @@ export enum ContainerArchitecture { } // VPC Export Names -export const VPC_ID_EXPORT_NAME = 'WorkshopVPC'; -export const VPC_CIDR_EXPORT_NAME = 'WorkshopVPCCidr'; -export const VPC_PRIVATE_SUBNETS_EXPORT_NAME = 'WorkshopVPCPrivateSubnets'; -export const VPC_PUBLIC_SUBNETS_EXPORT_NAME = 'WorkshopVPCPublicSubnets'; -export const VPC_ISOLATED_SUBNETS_EXPORT_NAME = 'WorkshopVPCIsolatedSubnets'; -export const VPC_AVAILABILITY_ZONES_EXPORT_NAME = 'WorkshopVPCAvailabilityZones'; -export const VPC_PRIVATE_SUBNET_CIDRS_EXPORT_NAME = 'WorkshopVPCPrivateSubnetCidrs'; -export const VPC_PUBLIC_SUBNET_CIDRS_EXPORT_NAME = 'WorkshopVPCPublicSubnetCidrs'; -export const VPC_ISOLATED_SUBNET_CIDRS_EXPORT_NAME = 'WorkshopVPCIsolatedSubnetCidrs'; +export const VPC_ID_EXPORT_NAME = 'public:WorkshopVPC'; +export const VPC_CIDR_EXPORT_NAME = 'public:WorkshopVPCCidr'; +export const VPC_PRIVATE_SUBNETS_EXPORT_NAME = 'public:WorkshopVPCPrivateSubnets'; +export const VPC_PUBLIC_SUBNETS_EXPORT_NAME = 'public:WorkshopVPCPublicSubnets'; +export const VPC_ISOLATED_SUBNETS_EXPORT_NAME = 'public:WorkshopVPCIsolatedSubnets'; +export const VPC_AVAILABILITY_ZONES_EXPORT_NAME = 'public:WorkshopVPCAvailabilityZones'; +export const VPC_PRIVATE_SUBNET_CIDRS_EXPORT_NAME = 'public:WorkshopVPCPrivateSubnetCidrs'; +export const VPC_PUBLIC_SUBNET_CIDRS_EXPORT_NAME = 'public:WorkshopVPCPublicSubnetCidrs'; +export const VPC_ISOLATED_SUBNET_CIDRS_EXPORT_NAME = 'public:WorkshopVPCIsolatedSubnetCidrs'; // SNS/SQS Export Names -export const SNS_TOPIC_ARN_EXPORT_NAME = 'WorkshopSNSTopicArn'; -export const SQS_QUEUE_ARN_EXPORT_NAME = 'WorkshopSQSQueueArn'; -export const SQS_QUEUE_URL_EXPORT_NAME = 'WorkshopSQSQueueUrl'; +export const SNS_TOPIC_ARN_EXPORT_NAME = 'public:WorkshopSNSTopicArn'; +export const SQS_QUEUE_ARN_EXPORT_NAME = 'public:WorkshopSQSQueueArn'; +export const SQS_QUEUE_URL_EXPORT_NAME = 'public:WorkshopSQSQueueUrl'; // ECS Export Names -export const ECS_CLUSTER_ARN_EXPORT_NAME = 'WorkshopECSClusterArn'; -export const ECS_CLUSTER_NAME_EXPORT_NAME = 'WorkshopECSClusterName'; -export const ECS_SECURITY_GROUP_ID_EXPORT_NAME = 'WorkshopECSSecurityGroupId'; +export const ECS_CLUSTER_ARN_EXPORT_NAME = 'public:WorkshopECSClusterArn'; +export const ECS_CLUSTER_NAME_EXPORT_NAME = 'public:WorkshopECSClusterName'; +export const ECS_SECURITY_GROUP_ID_EXPORT_NAME = 'public:WorkshopECSSecurityGroupId'; // EKS Export Names -export const EKS_CLUSTER_ARN_EXPORT_NAME = 'WorkshopEKSClusterArn'; -export const EKS_CLUSTER_NAME_EXPORT_NAME = 'WorkshopEKSClusterName'; -export const EKS_SECURITY_GROUP_ID_EXPORT_NAME = 'WorkshopEKSSecurityGroupId'; -export const EKS_KUBECTL_ROLE_ARN_EXPORT_NAME = 'WorkshopEKSKubectlRoleArn'; -export const EKS_OPEN_ID_CONNECT_PROVIDER_ARN_EXPORT_NAME = 'WorkshopEKSOpenIdConnectProviderArn'; -export const EKS_KUBECTL_SECURITY_GROUP_ID_EXPORT_NAME = 'WorkshopEKSKubectlSecurityGroupId'; -export const EKS_KUBECTL_LAMBDA_ROLE_ARN_EXPORT_NAME = 'WorkshopEKSKubectlLambdaRoleArn'; +export const EKS_CLUSTER_ARN_EXPORT_NAME = 'public:WorkshopEKSClusterArn'; +export const EKS_CLUSTER_NAME_EXPORT_NAME = 'public:WorkshopEKSClusterName'; +export const EKS_SECURITY_GROUP_ID_EXPORT_NAME = 'public:WorkshopEKSSecurityGroupId'; +export const EKS_KUBECTL_ROLE_ARN_EXPORT_NAME = 'public:WorkshopEKSKubectlRoleArn'; +export const EKS_OPEN_ID_CONNECT_PROVIDER_ARN_EXPORT_NAME = 'public:WorkshopEKSOpenIdConnectProviderArn'; +export const EKS_KUBECTL_SECURITY_GROUP_ID_EXPORT_NAME = 'public:WorkshopEKSKubectlSecurityGroupId'; +export const EKS_KUBECTL_LAMBDA_ROLE_ARN_EXPORT_NAME = 'public:WorkshopEKSKubectlLambdaRoleArn'; // Aurora Database Export Names -export const AURORA_CLUSTER_ARN_EXPORT_NAME = 'WorkshopAuroraClusterArn'; -export const AURORA_CLUSTER_ENDPOINT_EXPORT_NAME = 'WorkshopAuroraClusterEndpoint'; -export const AURORA_SECURITY_GROUP_ID_EXPORT_NAME = 'WorkshopAuroraSecurityGroupId'; -export const AURORA_ADMIN_SECRET_ARN_EXPORT_NAME = 'WorkshopAuroraAdminSecretArn'; //pragma: allowlist secret +export const AURORA_CLUSTER_ARN_EXPORT_NAME = 'public:WorkshopAuroraClusterArn'; +export const AURORA_CLUSTER_ENDPOINT_EXPORT_NAME = 'public:WorkshopAuroraClusterEndpoint'; +export const AURORA_SECURITY_GROUP_ID_EXPORT_NAME = 'public:WorkshopAuroraSecurityGroupId'; +export const AURORA_ADMIN_SECRET_ARN_EXPORT_NAME = 'private:WorkshopAuroraAdminSecretArn'; //pragma: allowlist secret // DynamoDB Export Names -export const DYNAMODB_TABLE_ARN_EXPORT_NAME = 'WorkshopDynamoDBTableArn'; -export const DYNAMODB_TABLE_NAME_EXPORT_NAME = 'WorkshopDynamoDBTableName'; +export const DYNAMODB_TABLE_ARN_EXPORT_NAME = 'public:WorkshopDynamoDBTableArn'; +export const DYNAMODB_TABLE_NAME_EXPORT_NAME = 'public:WorkshopDynamoDBTableName'; // OpenSearch Serverless Export Names -export const OPENSEARCH_COLLECTION_ARN_EXPORT_NAME = 'WorkshopOpenSearchCollectionArn'; -export const OPENSEARCH_COLLECTION_ID_EXPORT_NAME = 'WorkshopOpenSearchCollectionId'; -export const OPENSEARCH_COLLECTION_ENDPOINT_EXPORT_NAME = 'WorkshopOpenSearchCollectionEndpoint'; +export const OPENSEARCH_COLLECTION_ARN_EXPORT_NAME = 'public:WorkshopOpenSearchCollectionArn'; +export const OPENSEARCH_COLLECTION_ID_EXPORT_NAME = 'public:WorkshopOpenSearchCollectionId'; +export const OPENSEARCH_COLLECTION_ENDPOINT_EXPORT_NAME = 'public:WorkshopOpenSearchCollectionEndpoint'; // OpenSearch Application Export Names -export const OPENSEARCH_APPLICATION_ARN_EXPORT_NAME = 'WorkshopOpenSearchApplicationArn'; -export const OPENSEARCH_APPLICATION_ID_EXPORT_NAME = 'WorkshopOpenSearchApplicationId'; +export const OPENSEARCH_APPLICATION_ARN_EXPORT_NAME = 'public:WorkshopOpenSearchApplicationArn'; +export const OPENSEARCH_APPLICATION_ID_EXPORT_NAME = 'public:WorkshopOpenSearchApplicationId'; // OpenSearch Ingestion Pipeline Export Names -export const OPENSEARCH_PIPELINE_ARN_EXPORT_NAME = 'WorkshopOpenSearchPipelineArn'; -export const OPENSEARCH_PIPELINE_ENDPOINT_EXPORT_NAME = 'WorkshopOpenSearchPipelineEndpoint'; -export const OPENSEARCH_PIPELINE_ROLE_ARN_EXPORT_NAME = 'WorkshopOpenSearchPipelineRoleArn'; +export const OPENSEARCH_PIPELINE_ARN_EXPORT_NAME = 'public:WorkshopOpenSearchPipelineArn'; +export const OPENSEARCH_PIPELINE_ENDPOINT_EXPORT_NAME = 'public:WorkshopOpenSearchPipelineEndpoint'; +export const OPENSEARCH_PIPELINE_ROLE_ARN_EXPORT_NAME = 'public:WorkshopOpenSearchPipelineRoleArn'; // VPC Endpoint Export Names -export const VPC_ENDPOINT_APIGATEWAY_ID_EXPORT_NAME = 'WorkshopVPCEndpointApiGatewayId'; -export const VPC_ENDPOINT_DYNAMODB_ID_EXPORT_NAME = 'WorkshopVPCEndpointDynamoDbId'; -export const VPC_ENDPOINT_LAMBDA_ID_EXPORT_NAME = 'WorkshopVPCEndpointLambdaId'; -export const VPC_ENDPOINT_SERVICEDISCOVERY_ID_EXPORT_NAME = 'WorkshopVPCEndpointServiceDiscoveryId'; -export const VPC_ENDPOINT_DATA_SERVICEDISCOVERY_ID_EXPORT_NAME = 'WorkshopVPCEndpointDataServiceDiscoveryId'; -export const VPC_ENDPOINT_S3_ID_EXPORT_NAME = 'WorkshopVPCEndpointS3Id'; -export const VPC_ENDPOINT_SSM_ID_EXPORT_NAME = 'WorkshopVPCEndpointSSMId'; -export const VPC_ENDPOINT_EC2MESSAGES_ID_EXPORT_NAME = 'WorkshopVPCEndpointEC2MessagesId'; -export const VPC_ENDPOINT_SSMMESSAGES_ID_EXPORT_NAME = 'WorkshopVPCEndpointSSMMessagesId'; -export const VPC_ENDPOINT_SECRETSMANAGER_ID_EXPORT_NAME = 'WorkshopVPCEndpointSecretsManagerId'; -export const VPC_ENDPOINT_CLOUDWATCH_MONITORING_ID_EXPORT_NAME = 'WorkshopVPCEndpointCloudWatchMonitoringId'; -export const VPC_ENDPOINT_CLOUDWATCH_LOGS_ID_EXPORT_NAME = 'WorkshopVPCEndpointCloudWatchLogsId'; +export const VPC_ENDPOINT_APIGATEWAY_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointApiGatewayId'; +export const VPC_ENDPOINT_DYNAMODB_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointDynamoDbId'; +export const VPC_ENDPOINT_LAMBDA_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointLambdaId'; +export const VPC_ENDPOINT_SERVICEDISCOVERY_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointServiceDiscoveryId'; +export const VPC_ENDPOINT_DATA_SERVICEDISCOVERY_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointDataServiceDiscoveryId'; +export const VPC_ENDPOINT_S3_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointS3Id'; +export const VPC_ENDPOINT_SSM_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointSSMId'; +export const VPC_ENDPOINT_EC2MESSAGES_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointEC2MessagesId'; +export const VPC_ENDPOINT_SSMMESSAGES_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointSSMMessagesId'; +export const VPC_ENDPOINT_SECRETSMANAGER_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointSecretsManagerId'; +export const VPC_ENDPOINT_CLOUDWATCH_MONITORING_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointCloudWatchMonitoringId'; +export const VPC_ENDPOINT_CLOUDWATCH_LOGS_ID_EXPORT_NAME = 'public:WorkshopVPCEndpointCloudWatchLogsId'; // CloudMap Export Names -export const CLOUDMAP_NAMESPACE_ID_EXPORT_NAME = 'WorkshopCloudMapNamespaceId'; -export const CLOUDMAP_NAMESPACE_NAME_EXPORT_NAME = 'WorkshopCloudMapNamespaceName'; -export const CLOUDMAP_NAMESPACE_ARN_EXPORT_NAME = 'WorkshopCloudMapNamespaceArn'; +export const CLOUDMAP_NAMESPACE_ID_EXPORT_NAME = 'public:WorkshopCloudMapNamespaceId'; +export const CLOUDMAP_NAMESPACE_NAME_EXPORT_NAME = 'public:WorkshopCloudMapNamespaceName'; +export const CLOUDMAP_NAMESPACE_ARN_EXPORT_NAME = 'public:WorkshopCloudMapNamespaceArn'; // Assets Export Names -export const ASSETS_BUCKET_NAME_EXPORT_NAME = 'WorkshopAssetsBucketName'; -export const ASSETS_BUCKET_ARN_EXPORT_NAME = 'WorkshopAssetsBucketArn'; +export const ASSETS_BUCKET_NAME_EXPORT_NAME = 'public:WorkshopAssetsBucketName'; +export const ASSETS_BUCKET_ARN_EXPORT_NAME = 'public:WorkshopAssetsBucketArn'; // EventBridge Export Names -export const EVENTBUS_ARN_EXPORT_NAME = 'WorkshopEventBusArn'; -export const EVENTBUS_NAME_EXPORT_NAME = 'WorkshopEventBusName'; +export const EVENTBUS_ARN_EXPORT_NAME = 'public:WorkshopEventBusArn'; +export const EVENTBUS_NAME_EXPORT_NAME = 'public:WorkshopEventBusName'; // WAFv2 Export Names -export const WAFV2_REGIONAL_ACL_ARN_EXPORT_NAME = 'RegionalACLExportName'; -export const WAFV2_GLOABL_ACL_ARN_EXPORT_NAME = 'GlobalACLExportName'; +export const WAFV2_REGIONAL_ACL_ARN_EXPORT_NAME = 'public:RegionalACLExportName'; +export const WAFV2_GLOABL_ACL_ARN_EXPORT_NAME = 'public:GlobalACLExportName'; // SSM Parameter Names - Used across microservices export const SSM_PARAMETER_NAMES = { diff --git a/src/cdk/bin/local.ts b/src/cdk/bin/local.ts index 41300b6d..d3eec119 100644 --- a/src/cdk/bin/local.ts +++ b/src/cdk/bin/local.ts @@ -66,6 +66,7 @@ if (CUSTOM_ENABLE_WAF && process.env?.AWS_REGION != 'us-east-1') { region: 'us-east-1', account: process.env.AWS_ACCOUNT_ID, }, + tags: TAGS, }); const globalWaf = new GlobalWaf(globalWafStack, 'GlobalWaf', { logRetention: DEFAULT_RETENTION_DAYS, diff --git a/src/cdk/lib/constructs/waf.ts b/src/cdk/lib/constructs/waf.ts index 9b7941dc..e71e1f73 100644 --- a/src/cdk/lib/constructs/waf.ts +++ b/src/cdk/lib/constructs/waf.ts @@ -1,9 +1,10 @@ -import { RemovalPolicy } from 'aws-cdk-lib'; +import { Names, RemovalPolicy } from 'aws-cdk-lib'; import { CfnLoggingConfiguration, CfnWebACL } from 'aws-cdk-lib/aws-wafv2'; import { Construct } from 'constructs'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { PARAMETER_STORE_PREFIX } from '../../bin/environment'; +import { NagSuppressions } from 'cdk-nag'; export interface WorkshopWebAclProperties { logRetention?: RetentionDays; @@ -26,6 +27,7 @@ export class RegionalWaf extends Construct { const logGroup = new LogGroup(this, 'WAFv2RegionalLogGroup', { retention: properties.logRetention || RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, + logGroupName: 'aws-waf-logs-regional-' + Names.uniqueId(this), }); const webAcl = new CfnWebACL(this, 'WAFv2RegionalACL', { @@ -66,6 +68,13 @@ export class RegionalWaf extends Construct { logDestinationConfigs: [logGroup.logGroupArn], resourceArn: webAcl.attrArn, }); + + NagSuppressions.addResourceSuppressions(logGroup, [ + { + id: 'Workshop-CWL3', + reason: 'WAF Log group name must be prefixed with aws-waf-logs', + }, + ]); return webAcl; } @@ -100,6 +109,7 @@ export class GlobalWaf extends Construct { const logGroup = new LogGroup(this, 'WAFv2GlobalLogGroup', { retention: properties.logRetention || RetentionDays.ONE_WEEK, removalPolicy: RemovalPolicy.DESTROY, + logGroupName: 'aws-waf-logs-global-' + Names.uniqueId(this), }); const webAcl = new CfnWebACL(this, 'WAFv2GlobalACL', { defaultAction: { @@ -138,6 +148,12 @@ export class GlobalWaf extends Construct { logDestinationConfigs: [logGroup.logGroupArn], resourceArn: webAcl.attrArn, }); + NagSuppressions.addResourceSuppressions(logGroup, [ + { + id: 'Workshop-CWL3', + reason: 'WAF Log group name must be prefixed with aws-waf-logs', + }, + ]); return webAcl; } private createWafv2Outputs() { diff --git a/src/cdk/lib/pipeline.ts b/src/cdk/lib/pipeline.ts index c0f9a170..ef25b0e2 100644 --- a/src/cdk/lib/pipeline.ts +++ b/src/cdk/lib/pipeline.ts @@ -274,6 +274,37 @@ export class CDKPipeline extends Stack { }), ); + /** + * Add exports dashboard generation as the final pipeline stage + */ + const exportsDashboardWave = pipeline.addWave('ExportsDashboard'); + + const exportsDashboardStep = new CodeBuildStep('GenerateExportsDashboard', { + input: bucketSource, + commands: [ + `cd ${properties.workingFolder}`, + 'echo "Installing Python dependencies for exports generation..."', + 'pip3 install -r /scripts/requirements.txt', + 'echo "Generating CDK exports dashboard..."', + 'python3 scripts/manage-exports.py generate-dashboard', + 'echo "Exports dashboard generation completed"', + ], + buildEnvironment: { + buildImage: LinuxBuildImage.STANDARD_7_0, + privileged: false, + environmentVariables: { + NODE_VERSION: { + value: '22.x', + }, + CUSTOM_ENABLE_WAF: { + value: CUSTOM_ENABLE_WAF ? 'true' : 'false', + }, + }, + }, + }); + + exportsDashboardWave.addPost(exportsDashboardStep); + /** * Build the pipeline to add suppressions and customizations. * This is required before adding additional configurations. diff --git a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts index d398b378..3d9c6579 100644 --- a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts +++ b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts @@ -12,7 +12,7 @@ import { import { Construct } from 'constructs'; import { ManagedPolicy, PolicyDocument, Effect, PolicyStatement, StarPrincipal } from 'aws-cdk-lib/aws-iam'; import { ILayerVersion, LayerVersion } from 'aws-cdk-lib/aws-lambda'; -import { RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack, CfnOutput } from 'aws-cdk-lib'; import { BundlingOptions } from 'aws-cdk-lib/aws-lambda-nodejs'; import { EndpointType, LambdaRestApi, LogGroupLogDestination, MethodLoggingLevel } from 'aws-cdk-lib/aws-apigateway'; import { NagSuppressions } from 'cdk-nag'; @@ -149,6 +149,12 @@ export class StatusUpdatedService extends WokshopLambdaFunction { }), ), ); + + new CfnOutput(this, 'StatusUpdaterApiUrl', { + value: this.api.url, + exportName: 'StatusUpdaterApiUrl', + description: 'API Gateway URL for the pet status updater service', + }); } else { throw new Error('Service is not defined'); } diff --git a/src/cdk/lib/utils/utilities.ts b/src/cdk/lib/utils/utilities.ts index 84d62c4e..b5d5f284 100644 --- a/src/cdk/lib/utils/utilities.ts +++ b/src/cdk/lib/utils/utilities.ts @@ -149,9 +149,12 @@ export const Utilities = { } }, - createOuputs(scope: Construct, parameters: Map) { + createOutputs(scope: Construct, parameters: Map, descriptions?: Map) { for (const [key, value] of parameters.entries()) { - new CfnOutput(scope, key, { value: value }); + new CfnOutput(scope, key, { + value: value, + description: descriptions?.get(key) || `Output for ${key}`, + }); } }, }; diff --git a/src/cdk/lib/utils/workshop-nag-pack.ts b/src/cdk/lib/utils/workshop-nag-pack.ts index 639bd96b..3f5671ff 100644 --- a/src/cdk/lib/utils/workshop-nag-pack.ts +++ b/src/cdk/lib/utils/workshop-nag-pack.ts @@ -1,4 +1,4 @@ -import { CfnResource, Stack } from 'aws-cdk-lib'; +import { CfnResource, Stack, Token } from 'aws-cdk-lib'; import { IConstruct } from 'constructs'; import { NagPack, NagPackProps, NagRuleCompliance, NagRuleResult, NagMessageLevel, NagRules } from 'cdk-nag'; @@ -38,7 +38,7 @@ export class WorkshopNagPack extends NagPack { info: 'CloudWatch Log Groups should use dynamically generated names', explanation: 'Log groups with static names may cause conflicts when redeploying the stack. Use dynamically generated names instead.', - level: NagMessageLevel.ERROR, + level: NagMessageLevel.WARN, rule: this.checkCloudWatchLogGroupName, node: node, }); @@ -102,13 +102,25 @@ export class WorkshopNagPack extends NagPack { private checkCloudWatchLogGroupName = (node: CfnResource): NagRuleResult => { if (node.cfnResourceType === 'AWS::Logs::LogGroup') { - const logGroupName = NagRules.resolveIfPrimitive( - node, - (node as CfnResource & { logGroupName?: unknown }).logGroupName, - ); - if (logGroupName && typeof logGroupName === 'string' && !logGroupName.includes('Ref')) { + const nodeWithLogGroupName = node as CfnResource & { logGroupName?: unknown }; + + // If logGroupName property is not set at all, it's compliant + if (!('logGroupName' in nodeWithLogGroupName) || nodeWithLogGroupName.logGroupName === undefined) { + return NagRuleCompliance.COMPLIANT; + } + + const rawLogGroupName = nodeWithLogGroupName.logGroupName; + + // If logGroupName is a token/intrinsic function, it's non-compliant + if (Token.isUnresolved(rawLogGroupName)) { + return NagRuleCompliance.NON_COMPLIANT; + } + + // If it's a static string, it's non-compliant + if (typeof rawLogGroupName === 'string') { return NagRuleCompliance.NON_COMPLIANT; } + return NagRuleCompliance.COMPLIANT; } return NagRuleCompliance.NOT_APPLICABLE; diff --git a/src/cdk/scripts/README.md b/src/cdk/scripts/README.md new file mode 100644 index 00000000..98128e61 --- /dev/null +++ b/src/cdk/scripts/README.md @@ -0,0 +1,194 @@ +# CDK Exports Dashboard Generator + +A comprehensive solution for extracting, organizing, and displaying CDK stack exports in a professional, searchable web dashboard. + +## Overview + +This tool addresses the challenge of CDK generating dynamic resource names by providing a user-friendly dashboard to view all stack exports with: + +- **Multi-region support**: Automatically detects and aggregates exports from primary region + us-east-1 (when WAF is enabled) +- **Professional AWS-branded interface**: Clean, responsive design with official AWS styling +- **Advanced search**: Real-time search across export names, descriptions, stack names, and values +- **Smart categorization**: Automatic grouping by resource type (Networking, Database, Storage, etc.) +- **Easy navigation**: Copy-to-clipboard functionality, direct AWS Console links +- **Workshop compliance**: Includes required disclaimer for educational use + +## Features + +### Core Functionality + +- ✅ Extract exports from all CDK stacks across multiple regions +- ✅ Filter exports by prefix (e.g., "Workshop" exports only) +- ✅ Generate responsive HTML dashboard with search capabilities +- ✅ Upload to S3 with CloudFront integration +- ✅ Automatic integration into CDK pipeline + +### User Experience + +- ✅ Real-time search with highlighting +- ✅ Filter by category or stack name +- ✅ One-click copy to clipboard +- ✅ Direct links to AWS Console resources +- ✅ Mobile-responsive design +- ✅ Professional AWS branding + +## Usage + +### Command Line Interface + +```bash +# Generate complete dashboard (extract + HTML + upload) +python3 scripts/manage-exports.py generate-dashboard + +# Extract only Workshop-prefixed exports +python3 scripts/manage-exports.py generate-dashboard --filter-prefix Workshop + +# Extract exports to JSON file +python3 scripts/manage-exports.py extract --output exports.json + +# Generate HTML from existing JSON +python3 scripts/manage-exports.py generate-html --input exports.json --output dashboard.html + +# Upload existing HTML to S3 +python3 scripts/manage-exports.py upload-to-s3 --input dashboard.html --bucket my-bucket +``` + +### CDK Pipeline Integration + +The solution automatically integrates with your CDK pipeline as the final stage: + +1. **Automatic Execution**: Runs after all CDK stacks are deployed +2. **Multi-Region Detection**: Automatically includes us-east-1 when WAF is enabled +3. **S3 Upload**: Uploads dashboard to assets bucket with CloudFront distribution +4. **URL Generation**: CloudFormation template exposes dashboard URL as output + +### Environment Variables + +The script uses these environment variables for configuration: + +- `AWS_REGION`: Primary region for export scanning +- `CUSTOM_ENABLE_WAF`: Set to 'true' to include us-east-1 region +- `ASSETS_BUCKET_NAME`: S3 bucket for dashboard upload (auto-detected from exports) + +## File Structure + +``` +src/cdk/scripts/ +├── manage-exports.py # Main script with CLI interface +├── templates/ +│ └── exports-dashboard.j2 # Professional Jinja2 HTML template +└── README.md # This documentation +``` + +## Template Customization + +The HTML template can be customized by modifying `templates/exports-dashboard.j2`: + +- **Styling**: Update CSS variables for colors and branding +- **Layout**: Modify HTML structure and component arrangement +- **Functionality**: Enhance JavaScript for additional features +- **Content**: Customize text, disclaimers, and help information + +## Integration Details + +### CloudFormation Template Enhancement + +The solution enhances the existing CloudFormation deployment template: + +1. **Lambda Function**: Enhanced `rCDKOutputRetrieverFunction` constructs dashboard URL +2. **Output Addition**: New `oExportsDashboardUrl` output for easy access +3. **URL Construction**: Automatically detects CloudFront or falls back to S3 direct URL + +### CDK Pipeline Stage + +Added as final pipeline wave in `lib/pipeline.ts`: + +```typescript +const exportsDashboardWave = pipeline.addWave('ExportsDashboard'); +const exportsDashboardStep = new CodeBuildStep('GenerateExportsDashboard', { + commands: [ + 'pip3 install jinja2 boto3', + 'python3 scripts/manage-exports.py generate-dashboard --filter-prefix Workshop', + ], +}); +``` + +## Security & Compliance + +- **Workshop Disclaimer**: Prominent notice that content is for educational use only +- **No Sensitive Data**: Filters out internal AWS/CDK exports by default +- **Read-Only Access**: Script only reads CloudFormation exports, no modifications +- **Secure Upload**: Uses proper S3 permissions and HTTPS-only access + +## Troubleshooting + +### Common Issues + +**No exports found**: Check that CDK stacks have been deployed and exports exist + +```bash +aws cloudformation list-exports --region us-east-1 +``` + +**Permission denied**: Ensure AWS credentials have CloudFormation read access + +```bash +aws sts get-caller-identity +``` + +**Template not found**: Verify templates directory exists and contains `exports-dashboard.j2` + +```bash +ls -la scripts/templates/ +``` + +**S3 upload fails**: Check bucket exists and permissions allow PutObject + +```bash +aws s3 ls s3://your-assets-bucket/ +``` + +### Debug Mode + +Enable detailed logging by setting: + +```bash +export AWS_DEFAULT_REGION=us-east-1 +python3 scripts/manage-exports.py generate-dashboard --filter-prefix Workshop +``` + +## Development + +### Dependencies + +- Python 3.7+ +- boto3 (AWS SDK) +- jinja2 (Template engine) + +### Testing + +Test the script locally: + +```bash +# Install dependencies +pip3 install boto3 jinja2 + +# Test extraction (requires AWS credentials) +python3 scripts/manage-exports.py extract --filter-prefix Workshop + +# Test template rendering +python3 scripts/manage-exports.py generate-html --input exports.json +``` + +## License + +This tool is part of the AWS One Observability Workshop and follows the same Apache 2.0 license. + +## Contributing + +When contributing improvements: + +1. Test changes with various export configurations +2. Ensure mobile responsiveness is maintained +3. Validate AWS branding and disclaimer requirements +4. Update this README with new features or usage patterns diff --git a/src/cdk/scripts/manage-exports.py b/src/cdk/scripts/manage-exports.py new file mode 100755 index 00000000..0a5f6217 --- /dev/null +++ b/src/cdk/scripts/manage-exports.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +""" +CDK Stack Exports Management Tool + +This script extracts CloudFormation stack exports from CDK deployments, +generates a searchable HTML dashboard, and uploads it to S3 for easy access. + +Features: +- Multi-region export aggregation (primary region + us-east-1 for WAF) +- Professional HTML template with AWS branding +- Real-time search across export names, descriptions, and stack names +- S3 upload with CloudFront cache optimization +- Workshop disclaimer and security notices +""" + +import argparse +import boto3 +import json +import os +import sys +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional +from botocore.exceptions import ClientError, NoCredentialsError +from jinja2 import Environment, FileSystemLoader, select_autoescape + +# Configure logging first +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +# Load environment variables from .env file (like the CDK code does) +try: + from dotenv import load_dotenv + + # Look for .env file in the CDK directory structure + # First try: current directory (when running from src/cdk) + dotenv_path = os.path.join(os.getcwd(), ".env") + if os.path.exists(dotenv_path): + load_dotenv(dotenv_path) + logger.info("Loaded environment variables from %s", dotenv_path) + else: + # Second try: relative to this script's location (src/cdk/scripts -> src/cdk) + script_dir = os.path.dirname(os.path.abspath(__file__)) + dotenv_path = os.path.join(script_dir, "..", ".env") + if os.path.exists(dotenv_path): + load_dotenv(dotenv_path) + logger.info("Loaded environment variables from %s", dotenv_path) + else: + logger.info("No .env file found, using system environment variables only") + +except ImportError: + logger.warning( + "python-dotenv not available, using system environment variables only", + ) + + +class CDKExportsManager: + """Manages CDK stack exports extraction, HTML generation, and S3 upload.""" + + def __init__(self): + self.primary_region = os.environ.get("AWS_REGION", "us-east-1") + self.account_id = None + self.exports_data = [] + + def get_target_regions(self) -> List[str]: + """Determine which regions to scan for exports based on WAF configuration.""" + regions = [self.primary_region] + + # Check if WAF is enabled by looking for the environment variable + # or by checking for WAF-related exports in the current region + waf_enabled = os.environ.get("CUSTOM_ENABLE_WAF", "").lower() == "true" + + if not waf_enabled: + # Check for WAF exports in the current region + try: + cf_client = boto3.client( + "cloudformation", + region_name=self.primary_region, + ) + exports = cf_client.list_exports().get("Exports", []) + waf_enabled = any("WAF" in export.get("Name", "") for export in exports) + except Exception as e: + logger.warning(f"Could not check for WAF exports: {e}") + + if waf_enabled and "us-east-1" not in regions: + regions.append("us-east-1") + logger.info("WAF detected, including us-east-1 in export scan") + + logger.info(f"Scanning regions: {regions}") + return regions + + def extract_exports( + self, + filter_prefix: Optional[str] = None, + exclude_internal: bool = True, + ) -> List[Dict]: + """ + Extract CloudFormation exports from all target regions. + + Args: + filter_prefix: Only include exports with names starting with this prefix + exclude_internal: Exclude AWS internal exports (AWS::, CDK::, etc.) + + Returns: + List of export dictionaries with metadata + """ + all_exports = [] + regions = self.get_target_regions() + + for region in regions: + logger.info(f"Extracting exports from region: {region}") + + try: + cf_client = boto3.client("cloudformation", region_name=region) + + # Get account ID from STS if not already retrieved + if not self.account_id: + sts_client = boto3.client("sts", region_name=region) + self.account_id = sts_client.get_caller_identity()["Account"] + + # Paginate through exports + paginator = cf_client.get_paginator("list_exports") + + for page in paginator.paginate(): + for export in page.get("Exports", []): + export_name = export.get("Name", "") + export_value = export.get("Value", "") + exporting_stack_id = export.get("ExportingStackId", "") + + # Extract stack name from stack ID + stack_name = self._extract_stack_name(exporting_stack_id) + + # Apply filters + if filter_prefix and not export_name.startswith(filter_prefix): + continue + + if exclude_internal and self._is_internal_export(export_name): + continue + + # Get additional stack metadata + stack_info = self._get_stack_info(cf_client, stack_name) + + export_data = { + "exportName": export_name, + "exportValue": export_value, + "stackName": stack_name, + "stackId": exporting_stack_id, + "region": region, + "description": stack_info.get("description", ""), + "stackStatus": stack_info.get("status", ""), + "creationTime": stack_info.get("creation_time", ""), + "tags": stack_info.get("tags", {}), + "category": self._categorize_export(export_name), + "isUrl": self._is_url_value(export_value), + "consoleUrl": self._get_console_url( + region, + stack_name, + export_name, + export_value, + ), + } + + all_exports.append(export_data) + + except ClientError as e: + logger.error("Error extracting exports from %s: %s", region, e) + continue + except Exception as e: + logger.error("Unexpected error in %s: %s", region, e) + continue + + # Sort exports by category, then by stack name, then by export name + all_exports.sort(key=lambda x: (x["category"], x["stackName"], x["exportName"])) + + logger.info( + "Extracted %d exports from %d regions", + len(all_exports), + len(regions), + ) + self.exports_data = all_exports + return all_exports + + def _extract_stack_name(self, stack_id: str) -> str: + """Extract stack name from CloudFormation stack ID/ARN.""" + if not stack_id: + return "Unknown" + + # Stack ID format: arn:aws:cloudformation:region:account:stack/stack-name/uuid + if stack_id.startswith("arn:aws:cloudformation:"): + parts = stack_id.split("/") + if len(parts) >= 2: + return parts[1] + + # For non-ARN formats, try to extract meaningful name + return stack_id.split("/")[-1] if "/" in stack_id else stack_id + + def _is_internal_export(self, export_name: str) -> bool: + """Check if an export is AWS/CDK internal.""" + internal_prefixes = ["AWS::", "CDK::", "cdk-", "CdkBootstrap", "StagingBucket"] + return any(export_name.startswith(prefix) for prefix in internal_prefixes) + + def _categorize_export(self, export_name: str) -> str: + """Categorize export based on prefix and naming patterns.""" + # First check for prefix-based categorization + if ":" in export_name: + prefix = export_name.split(":", 1)[0] + if prefix == "public": + # For public exports, use detailed categorization + name_lower = export_name.lower() + + if any( + term in name_lower for term in ["vpc", "subnet", "cidr", "network"] + ): + return "Networking" + elif any( + term in name_lower + for term in ["database", "db", "rds", "aurora", "dynamo"] + ): + return "Database" + elif any(term in name_lower for term in ["bucket", "s3", "assets"]): + return "Storage" + elif any( + term in name_lower for term in ["cluster", "ecs", "eks", "compute"] + ): + return "Compute" + elif any( + term in name_lower for term in ["api", "gateway", "endpoint", "url"] + ): + return "API" + elif any( + term in name_lower for term in ["security", "role", "policy", "waf"] + ): + return "Security" + elif any( + term in name_lower + for term in ["monitor", "log", "metric", "opensearch"] + ): + return "Observability" + else: + return "Other" + elif prefix == "private": + return "Private" + else: + # Unknown prefix, treat as other + return "Other" + else: + # No prefix (no colon), categorize as internal-cdk + return "internal-cdk" + + def _is_url_value(self, value: str) -> bool: + """Check if export value appears to be a URL.""" + return value.startswith(("http://", "https://")) + + def _get_console_url( + self, + region: str, + stack_name: str, + export_name: str, + export_value: str, + ) -> str: + """Generate AWS Console URL for the export's resource.""" + base_url = "https://console.aws.amazon.com/cloudformation/home" + return f"{base_url}?region={region}#/stacks/stackinfo?stackId={stack_name}" + + def _get_stack_info(self, cf_client, stack_name: str) -> Dict: + """Get additional information about a CloudFormation stack.""" + try: + response = cf_client.describe_stacks(StackName=stack_name) + stack = response["Stacks"][0] + + # Convert datetime objects to strings + creation_time = stack.get("CreationTime") + if creation_time: + creation_time = creation_time.isoformat() + + # Process tags + tags = {} + for tag in stack.get("Tags", []): + tags[tag["Key"]] = tag["Value"] + + return { + "description": stack.get("Description", ""), + "status": stack.get("StackStatus", ""), + "creation_time": creation_time, + "tags": tags, + } + except Exception as e: + logger.debug(f"Could not get stack info for {stack_name}: {e}") + return {} + + def generate_html(self, template_path: Optional[str] = None) -> str: + """ + Generate HTML dashboard from exports data. + + Args: + template_path: Path to custom Jinja2 template file + + Returns: + Generated HTML string + """ + if not self.exports_data: + raise ValueError("No exports data available. Run extract_exports() first.") + + # Set up Jinja2 environment + template_dir = template_path or os.path.join( + os.path.dirname(__file__), + "templates", + ) + env = Environment( + loader=FileSystemLoader(template_dir), + autoescape=select_autoescape(["html", "xml"]), + ) + + # Load template + try: + template = env.get_template("exports-dashboard.j2") + except Exception as e: + logger.error(f"Could not load template: {e}") + # Fall back to built-in template + template = env.from_string(self._get_builtin_template()) + + # Prepare template context + context = { + "exports": self.exports_data, + "generated_at": datetime.now(timezone.utc).isoformat(), + "total_exports": len(self.exports_data), + "regions": list({export["region"] for export in self.exports_data}), + "categories": list({export["category"] for export in self.exports_data}), + "stacks": list({export["stackName"] for export in self.exports_data}), + "account_id": self.account_id or "Unknown", + } + + # Generate HTML + html_content = template.render(**context) + logger.info("Generated HTML dashboard with %d exports", len(self.exports_data)) + + return html_content + + def upload_to_s3(self, html_content: str, bucket_name: Optional[str] = None) -> str: + """ + Upload HTML dashboard to S3. + + Args: + html_content: Generated HTML content + bucket_name: S3 bucket name (defaults to assets bucket from exports) + + Returns: + S3 URL of uploaded file + """ + if not bucket_name: + # Try to get bucket name from exports or environment + bucket_name = self._get_assets_bucket_name() + + if not bucket_name: + raise ValueError( + "No S3 bucket specified and could not determine assets bucket", + ) + + s3_client = boto3.client("s3") + key = "workshop-exports/index.html" + + try: + # Upload with proper content type and caching headers + s3_client.put_object( + Bucket=bucket_name, + Key=key, + Body=html_content.encode("utf-8"), + ContentType="text/html; charset=utf-8", + CacheControl="max-age=300, must-revalidate", # 5-minute cache + Metadata={ + "generated-at": datetime.now(timezone.utc).isoformat(), + "total-exports": str(len(self.exports_data)), + "generator": "cdk-exports-manager", + }, + ) + + # Generate public URL + s3_url = f"https://{bucket_name}.s3.amazonaws.com/{key}" + + # Try to get CloudFront URL if available + cloudfront_url = self._get_cloudfront_url(bucket_name, key) + + logger.info(f"Uploaded exports dashboard to S3: {s3_url}") # noqa: E501 + if cloudfront_url: + logger.info(f"CloudFront URL: {cloudfront_url}") # noqa: E501 + return cloudfront_url + + return s3_url + + except ClientError as e: + logger.error(f"Failed to upload to S3: {e}") + raise + + def _get_assets_bucket_name(self) -> Optional[str]: + """Try to determine the assets bucket name from exports or environment.""" + # Check environment variable first + bucket_name = os.environ.get("ASSETS_BUCKET_NAME") + if bucket_name: + return self._extract_bucket_name_from_arn(bucket_name) + + # Look for assets bucket in exports + for export in self.exports_data: + if ( + "AssetsBucket" in export["exportName"] + or "assets" in export["exportName"].lower() + ): + return self._extract_bucket_name_from_arn(export["exportValue"]) + + return None + + def _extract_bucket_name_from_arn(self, bucket_identifier: str) -> str: + """Extract bucket name from S3 ARN or return as-is if already a bucket name.""" + if not bucket_identifier: + return bucket_identifier + + # Check if it's an S3 ARN: arn:aws:s3:::bucket-name + if bucket_identifier.startswith("arn:aws:s3:::"): + # Extract bucket name after the last ::: + return bucket_identifier.split(":::")[-1] + + # If not an ARN, assume it's already a bucket name + return bucket_identifier + + def _get_cloudfront_url(self, bucket_name: str, key: str) -> Optional[str]: + """Try to get CloudFront distribution URL for the S3 bucket.""" + try: + # Look specifically for WorkshopCloudFrontDomain export first + for export in self.exports_data: + if export["exportName"] == "WorkshopCloudFrontDomain": + base_url = export["exportValue"] + if base_url.startswith("https://"): + return f"{base_url.rstrip('/')}/{key}" + else: + # Add https if not present + return f"https://{base_url.rstrip('/')}/{key}" + + # Fallback: Look for other CloudFront-related exports + for export in self.exports_data: + if ( + "cloudfront" in export["exportName"].lower() + or "distribution" in export["exportName"].lower() + ): + base_url = export["exportValue"] + if base_url.startswith("https://"): + return f"{base_url.rstrip('/')}/{key}" + else: + return f"https://{base_url.rstrip('/')}/{key}" + + return None + except Exception as e: + logger.debug(f"Could not determine CloudFront URL: {e}") # noqa: E501 + return None + + def _get_builtin_template(self) -> str: + """Return built-in HTML template if external template is not available.""" + return """ # noqa: E501 + + + + + + CDK Stack Exports Dashboard + + + + + + + + + + """ + + +def main(): + """Main CLI entry point.""" + parser = argparse.ArgumentParser( + description="CDK Stack Exports Management Tool", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Extract all exports and generate dashboard + python manage-exports.py generate-dashboard + + # Extract only Workshop-prefixed exports + python manage-exports.py extract --filter-prefix Workshop + + # Generate HTML with custom template + python manage-exports.py generate-html --template my-template.j2 + + # Upload to specific S3 bucket + python manage-exports.py upload-to-s3 --bucket my-bucket + """, + ) + + subparsers = parser.add_subparsers(dest="command", help="Commands") + + # Extract command + extract_parser = subparsers.add_parser("extract", help="Extract exports to JSON") + extract_parser.add_argument( + "--output", + "-o", + default="exports.json", + help="Output JSON file path", + ) + extract_parser.add_argument( + "--filter-prefix", + help="Only include exports with this prefix", + ) + extract_parser.add_argument( + "--include-internal", + action="store_true", + help="Include AWS/CDK internal exports", + ) + + # Generate HTML command + html_parser = subparsers.add_parser("generate-html", help="Generate HTML dashboard") + html_parser.add_argument( + "--input", + "-i", + default="exports.json", + help="Input JSON file path", + ) + html_parser.add_argument( + "--output", + "-o", + default="exports-dashboard.html", + help="Output HTML file path", + ) + html_parser.add_argument("--template", help="Custom Jinja2 template path") + + # Upload command + upload_parser = subparsers.add_parser("upload-to-s3", help="Upload HTML to S3") + upload_parser.add_argument( + "--input", + "-i", + default="exports-dashboard.html", + help="Input HTML file path", + ) + upload_parser.add_argument("--bucket", help="S3 bucket name") + + # Generate dashboard (all-in-one) command + dashboard_parser = subparsers.add_parser( + "generate-dashboard", + help="Extract, generate, and upload dashboard", + ) + dashboard_parser.add_argument( + "--filter-prefix", + default="Workshop", + help="Export name prefix filter", + ) + dashboard_parser.add_argument("--bucket", help="S3 bucket name") + dashboard_parser.add_argument("--template", help="Custom template path") + + args = parser.parse_args() + + if not args.command: + parser.print_help() + sys.exit(1) + + try: + manager = CDKExportsManager() + + if args.command == "extract": + exports = manager.extract_exports( + filter_prefix=args.filter_prefix, + exclude_internal=not args.include_internal, + ) + + with open(args.output, "w") as f: + json.dump(exports, f, indent=2, default=str) + + logger.info(f"Exported {len(exports)} entries to {args.output}") + + elif args.command == "generate-html": + # Load exports from JSON + with open(args.input) as f: + manager.exports_data = json.load(f) + + html_content = manager.generate_html(args.template) + + with open(args.output, "w", encoding="utf-8") as f: + f.write(html_content) + + logger.info(f"Generated HTML dashboard: {args.output}") + + elif args.command == "upload-to-s3": + with open(args.input, encoding="utf-8") as f: + html_content = f.read() + + url = manager.upload_to_s3(html_content, args.bucket) + logger.info(f"Dashboard available at: {url}") + print(url) # Output URL for use in scripts + + elif args.command == "generate-dashboard": + # All-in-one: extract, generate, and upload + logger.info("Starting complete dashboard generation...") + + exports = manager.extract_exports( + filter_prefix=args.filter_prefix, + exclude_internal=True, + ) + + if not exports: + logger.warning("No exports found matching criteria") + sys.exit(1) + + html_content = manager.generate_html(args.template) + url = manager.upload_to_s3(html_content, args.bucket) + + logger.info("Dashboard generation complete!") + logger.info("Found %d exports", len(exports)) + logger.info("Dashboard URL: %s", url) + print(url) # Output for pipeline use + + except NoCredentialsError: + logger.error("AWS credentials not found. Please configure your credentials.") + sys.exit(1) + except Exception as e: + logger.error(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/cdk/scripts/requirements.txt b/src/cdk/scripts/requirements.txt new file mode 100644 index 00000000..36c99608 --- /dev/null +++ b/src/cdk/scripts/requirements.txt @@ -0,0 +1,18 @@ +# CDK Scripts Python Dependencies +# Required for manage-exports.py script + +# AWS SDK for Python - CloudFormation, S3, STS operations +boto3>=1.28.0 + +# Template engine for HTML dashboard generation +jinja2>=3.1.0 + +# Environment variable loading from .env files +# Matches the pattern used by CDK TypeScript code +python-dotenv>=1.0.0 + +# Additional dependencies that might be useful for future scripts +# JSON handling is built into Python 3 +# argparse is built into Python 3 +# logging is built into Python 3 +# datetime is built into Python 3 diff --git a/src/cdk/scripts/templates/exports-dashboard.j2 b/src/cdk/scripts/templates/exports-dashboard.j2 new file mode 100644 index 00000000..09d248cf --- /dev/null +++ b/src/cdk/scripts/templates/exports-dashboard.j2 @@ -0,0 +1,739 @@ + + + + + + CDK Stack Exports Dashboard - AWS Workshop + + + + + + + +
+
+
+
+
+ +
+

CDK Stack Exports Dashboard

+

AWS One Observability Workshop

+
+
+
+
+ + Account: {{ account_id }}
+ Generated: {{ generated_at[:19] | replace('T', ' ') }} UTC +
+
+
+
+
+ +
+ + + + +
+
+
+
{{ total_exports }}
+
Total Exports
+
+
+
+
+
{{ stacks | length }}
+
CDK Stacks
+
+
+
+
+
{{ regions | length }}
+
AWS Regions
+
+
+
+
+
{{ categories | length }}
+
Categories
+
+
+
+ + +
+

+ Search Exports +

+
+
+ +
+
+ + +
+
Filter by Category:
+ + {% for category in categories | sort %} + {% if category != 'Private' and category != 'internal-cdk' %} + + {% endif %} + {% endfor %} + +
Advanced Options:
+ + +
Filter by Stack:
+ + {% for stack in stacks | sort %} + + {% endfor %} +
+ + +
+ + Showing {{ total_exports }} of {{ total_exports }} exports + +
+
+ + +
+ {% for export in exports %} +
+ + +
+
+
+
{{ export.exportName }}
+
+ + {{ export.category }} + + {{ export.stackName }} + {{ export.region }} +
+
+ +
+
+ + +
+ {% if export.description %} +
+ Description: {{ export.description }} +
+ {% endif %} + +
+ Export Value: +
+
+ {% if export.isUrl %} + {{ export.exportValue }} + {% else %} + {{ export.exportValue }} + {% endif %} + +
+ + {% if export.tags %} +
+ + Stack Tags: + {% for key, value in export.tags.items() %} + {{ key }}: {{ value }} + {% endfor %} + +
+ {% endif %} +
+
+ {% endfor %} +
+ + + +
+ + + + + + + + + diff --git a/src/cdk/scripts/test-dashboard.html b/src/cdk/scripts/test-dashboard.html new file mode 100644 index 00000000..0398190a --- /dev/null +++ b/src/cdk/scripts/test-dashboard.html @@ -0,0 +1,440 @@ + + + + + + CDK Stack Exports Dashboard - AWS Workshop + + + + + + + +
+
+
+
+
+ +
+

CDK Stack Exports Dashboard

+

AWS One Observability Workshop

+
+
+
+
+ + Account: 123456789012
+ Generated: 2025-10-24 12:30 UTC +
+
+
+
+
+ +
+ + + + +
+
+
+
42
+
Total Exports
+
+
+
+
+
8
+
CDK Stacks
+
+
+
+
+
2
+
AWS Regions
+
+
+
+
+
6
+
Categories
+
+
+
+ + +
+

Search Exports

+
+
+ +
+
+ + +
+
Filter by Category:
+ + + + + + + + +
Advanced Options:
+ +
+
+ + +
+
+
+
+
public:WorkshopVPC
+
+ Networking + CoreStack + us-east-1 +
+
+ +
+
+
+
Description: VPC ID for the workshop network
+
+ Export Value: +
+
+ vpc-0123456789abcdef0 + +
+
+
+ +
+
+
+
+
public:WorkshopDynamoDBTableArn
+
+ Database + StorageStack + us-east-1 +
+
+ +
+
+
+
+ Description: ARN of the DynamoDB table for pet adoption data +
+
+ Export Value: +
+
+ arn:aws:dynamodb:us-east-1:123456789012:table/PetAdoptionTable + +
+
+
+
+ + + + + diff --git a/src/templates/codebuild-deployment-template.yaml b/src/templates/codebuild-deployment-template.yaml index 634a7054..3b764718 100644 --- a/src/templates/codebuild-deployment-template.yaml +++ b/src/templates/codebuild-deployment-template.yaml @@ -495,6 +495,41 @@ Resources: return outputs = {o['ExportName']: o['OutputValue'] for o in response['Stacks'][0].get('Outputs', []) if 'ExportName' in o} + + # Construct exports dashboard URL from CloudFront domain and assets bucket + dashboard_url = None + cloudfront_domain = None + assets_bucket = None + + # Look for WorkshopCloudFrontDomain export first + for export_name, export_value in outputs.items(): + if export_name == 'WorkshopCloudFrontDomain': + if export_value.startswith('https://'): + cloudfront_domain = export_value.rstrip('/') + else: + cloudfront_domain = f"https://{export_value.rstrip('/')}" + break + + # Find assets bucket + for export_name, export_value in outputs.items(): + if 'AssetsBucket' in export_name or 'assets' in export_name.lower(): + # Extract bucket name from ARN if needed + if export_value.startswith('arn:aws:s3:::'): + assets_bucket = export_value.split(':::')[-1] + else: + assets_bucket = export_value + break + + # Construct dashboard URL + if cloudfront_domain: + dashboard_url = f"{cloudfront_domain}/workshop-exports/index.html" + elif assets_bucket: + dashboard_url = f"https://{assets_bucket}.s3.amazonaws.com/workshop-exports/index.html" + + # Add the dashboard URL to outputs if we found it + if dashboard_url: + outputs['ExportsDashboardUrl'] = dashboard_url + cfnresponse.send(event, context, cfnresponse.SUCCESS, outputs) except Exception as e: cfnresponse.send(event, context, cfnresponse.FAILED, {}, str(e)) @@ -1769,3 +1804,9 @@ Outputs: oPetSiteUrl: Description: PetSite URL from CDK stack Value: !GetAtt rCDKOutputs.PetSiteUrl + + oExportsDashboardUrl: + Description: URL to access the CDK exports dashboard + Value: !GetAtt rCDKOutputs.ExportsDashboardUrl + Export: + Name: !Sub ${AWS::StackName}-ExportsDashboardUrl From 2cb402ae9969b5fcb4248603757c03680140f5a0 Mon Sep 17 00:00:00 2001 From: Rafael Pereyra <31078199+rafaelpereyra@users.noreply.github.com> Date: Sat, 25 Oct 2025 12:52:53 -0400 Subject: [PATCH 3/3] Feat/codeconnection (#448) * feat: add CodeConnection and Parameter Store integration - Add CodeConnection support for GitHub integration as alternative to S3 source - Implement Parameter Store configuration management for centralized config - Update CONTRIBUTING.md formatting (bullet points, spacing, emphasis) - Add comprehensive documentation for new integration features - Create reusable configuration retrieval script for pipeline steps - Update CloudFormation template with new parameters and IAM permissions - Modify CDK pipeline to support conditional source selection - Add fallback mechanisms for backward compatibility * feat: enhance configuration flexibility and add troubleshooting docs - Add troubleshooting section for CDK bootstrap stack deletion issues - Support environment variables for Parameter Store base path configuration - Add CodeConnection ARN support for GitHub integration - Update workshop template with consistent parameter defaults - Enable conditional source configuration (CodeConnection vs S3) in local deployment * refactor: simplify parameter store configuration management Modified parameter storage approach in AWS Systems Manager Parameter Store from individual key-value parameters to a single parameter containing the complete .env file content. Updated the retrieve-config.sh script to fetch a single parameter instead of using get-parameters-by-path, and modified the CodeBuild deployment template to store the entire .env file as one parameter rather than splitting it into multiple parameters. * feat: implement single parameter approach for Parameter Store integration Updated CodeConnection and Parameter Store integration with single parameter approach. Modified documentation to reflect new CloudFormation-managed parameter creation, updated CDK pipeline to use single parameter path with stack name, and enhanced CodeBuild template to create Parameter Store parameter as CloudFormation resource instead of manual creation. * fix: deployment issues * fix: added tags to initial stack * fix: pipeline error * fix: update environment validation and opensearch pipeline logging - Modified environment variable validation to accept either CONFIG_BUCKET or CODE_CONNECTION_ARN - Reordered CloudWatch log group creation before IAM role definition in OpenSearch pipeline - Fixed log group ARN references in IAM policies to use correct log group name * fix: rolled back log name for opensearch * fix: publish export error * fix: missing permissions for dashboard * fix: unterminated quoete * fix: missing permissions * fix: added shell for export dashboard * feat: enhance CDK infrastructure and deployment scripts - Updated constants configuration in bin/constants.ts - Enhanced asset constructs and petsite microservice - Modified pipeline configuration and status updater function - Significantly expanded manage-exports.py script with 654 additions - Enhanced retrieve-config.sh script with 318 additions - Total changes: 860 additions, 138 deletions across 7 files * feat: added debug flag for scripts * fix: script error handling logic * feat: improved dashboard * fix: missing permissions for dashboard * cicd: removed debug flag * fix: broad permissions for putobject * feat: improved console access links --- .secrets.baseline | 6 +- CONTRIBUTING.md | 154 +++- ...econnection-parameter-store-integration.md | 208 +++++ .../nodejs/{ => node_modules}/package.json | 0 .../canaries/petsite-canary/nodejs/index.js | 121 --- .../nodejs/node_modules/index.js | 212 +++-- .../nodejs/{ => node_modules}/package.json | 0 src/cdk/bin/constants.ts | 9 + src/cdk/bin/environment.ts | 6 +- src/cdk/bin/local.ts | 25 +- src/cdk/bin/workshop.ts | 14 +- src/cdk/lib/constructs/assets.ts | 6 +- src/cdk/lib/constructs/opensearch-pipeline.ts | 25 +- src/cdk/lib/microservices/petsite.ts | 4 +- src/cdk/lib/pipeline.ts | 184 +++- .../status-updater/status-updater.ts | 4 +- src/cdk/lib/stages/containers.ts | 67 +- src/cdk/scripts/manage-exports.py | 853 +++++++++++++++--- src/cdk/scripts/retrieve-config.sh | 292 ++++++ .../scripts/templates/exports-dashboard.j2 | 4 +- .../codebuild-deployment-template.yaml | 110 ++- 21 files changed, 1854 insertions(+), 450 deletions(-) create mode 100644 docs/codeconnection-parameter-store-integration.md rename src/applications/canaries/housekeeping/nodejs/{ => node_modules}/package.json (100%) delete mode 100644 src/applications/canaries/petsite-canary/nodejs/index.js rename src/applications/canaries/petsite-canary/nodejs/{ => node_modules}/package.json (100%) create mode 100755 src/cdk/scripts/retrieve-config.sh diff --git a/.secrets.baseline b/.secrets.baseline index aeda32c6..2fd050ab 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -90,6 +90,10 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -246,5 +250,5 @@ } ] }, - "generated_at": "2025-10-22T23:26:53Z" + "generated_at": "2025-10-24T20:13:04Z" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7da333bc..ea0ef685 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,7 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 --> + # Contributing Guidelines Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional @@ -10,7 +11,6 @@ documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. - ## Reporting Bugs/Feature Requests We welcome you to use the GitHub issue tracker to report bugs or suggest features. @@ -18,16 +18,16 @@ We welcome you to use the GitHub issue tracker to report bugs or suggest feature When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment - +- A reproducible test case or series of steps +- The version of our code being used +- Any modifications you've made relevant to the bug +- Anything unusual about your environment or deployment ## Contributing via Pull Requests + Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: -1. You are working against the latest source on the *master* branch. +1. You are working against the latest source on the _master_ branch. 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. @@ -43,28 +43,29 @@ To send us a pull request, please: GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). - ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. ## Code of Conduct + This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact opensource-codeofconduct@amazon.com with any additional questions or comments. - ## Security Scanning and Pre-commit Hooks This project uses pre-commit hooks to ensure code quality and security. The following hooks are configured: ### Security Hooks + - **python-safety-dependencies-check**: Scans Python dependencies for known security vulnerabilities - **detect-secrets**: Prevents secrets from being committed to the repository - **detect-private-key**: Detects private keys in code - **detect-aws-credentials**: Prevents AWS credentials from being committed ### Code Quality Hooks + - **commitizen**: Enforces conventional commit message format - **check-json**: Validates JSON file syntax - **check-yaml**: Validates YAML file syntax @@ -84,18 +85,20 @@ This project uses pre-commit hooks to ensure code quality and security. The foll ### Prerequisites 1. **Install git-remote-s3** (required for pushing initial repo to S3 for container pipelines): - ```bash - pip install git-remote-s3 - ``` + + ```bash + pip install git-remote-s3 + ``` 2. **Install dependencies** from the root of the repository: - ```bash - npm install - ``` + ```bash + npm install + ``` ### Installing Pre-commit Hooks **Mac:** + ```bash # Use pip instead of brew to avoid old version issues pip install pre-commit @@ -104,6 +107,7 @@ pre-commit install --hook-type commit-msg ``` **Windows:** + ```bash pip install pre-commit pre-commit install @@ -115,10 +119,12 @@ pre-commit install --hook-type commit-msg For faster development without waiting for the pipeline, you can use the local CDK application located in `bin/local.ts`. This deploys resources directly using CDK commands. **Prerequisites:** + - Authenticate with your target AWS account - **IMPORTANT:** Run the deploy check script first (see Deployment Scripts section below) **Usage:** + ```bash # List available stacks cdk -a "npx ts-node bin/local.ts" list @@ -131,6 +137,7 @@ cdk -a "npx ts-node bin/local.ts" ``` Example commands: + ```bash # Show differences cdk -a "npx ts-node bin/local.ts" diff @@ -148,21 +155,94 @@ cdk -a "npx ts-node bin/local.ts" destroy The script validates your environment and prepares the repository for deployment. **Setup:** + 1. Copy `src/cdk/.env.sample` to `src/cdk/.env` 2. Update the `.env` file with your AWS account details: - - `CONFIG_BUCKET`: Your S3 bucket name - - `BRANCH_NAME`: Your git branch name - - `AWS_ACCOUNT_ID`: Your AWS account ID - - `AWS_REGION`: Your target AWS region - - `EKS_CLUSTER_ACCESS_ROLE_NAME`: Name of the role that will receive ClusterAdmin access on the EKS Cluster (optional) - - `ENABLE_PET_FOOD_AGENT`: Set to `true` to enable AgentCore deployment (optional) + - `CONFIG_BUCKET`: Your S3 bucket name + - `BRANCH_NAME`: Your git branch name + - `AWS_ACCOUNT_ID`: Your AWS account ID + - `AWS_REGION`: Your target AWS region + - `EKS_CLUSTER_ACCESS_ROLE_NAME`: Name of the role that will receive ClusterAdmin access on the EKS Cluster (optional) + - `ENABLE_PET_FOOD_AGENT`: Set to `true` to enable AgentCore deployment (optional) + +### CodeConnection and Parameter Store Integration + +The project supports optional CodeConnection integration for GitHub source and Parameter Store for configuration management. These features provide enhanced security and flexibility for deployments. + +**Additional Configuration Parameters:** + +When using the CloudFormation deployment template (`src/templates/codebuild-deployment-template.yaml`), you can optionally configure: + +- `pCodeConnectionArn`: ARN of an existing AWS CodeStar Connection to GitHub + - When provided, the pipeline will use CodeConnection as the source instead of S3 bucket + - Format: `arn:aws:codeconnections:region:account:connection/connection-id` + - If not provided, the pipeline falls back to S3 bucket source (default behavior) + +- `pParameterStoreBasePath`: Base path prefix for storing configuration in Parameter Store + - Used to store workshop configuration parameters (replaces .env file approach) + - Format: `/your-prefix/` (must start and end with `/`) + - Example: `/one-observability-workshop/` + - Parameters are stored as: `${basePath}parameter-name` + +**Local Development with CodeConnection:** + +For local development when using CodeConnection: + +1. **Create CodeConnection** (if not exists): + + ```bash + # Create connection via AWS Console or CLI + aws codeconnections create-connection \ + --provider-type GitHub \ + --connection-name one-observability-demo + ``` + +2. **Configure .env file** with CodeConnection details: + + ```bash + # Add to src/cdk/.env + CODE_CONNECTION_ARN=arn:aws:codeconnections:us-east-1:123456789012:connection/abc123 + PARAMETER_STORE_BASE_PATH=/one-observability-workshop/ + ``` + +3. **Deploy with CodeConnection**: + ```bash + # The local CDK app will automatically use CodeConnection if ARN is provided + cdk -a "npx ts-node bin/local.ts" deploy --all + ``` + +**Parameter Store Configuration:** + +When using Parameter Store, configuration values are automatically retrieved during pipeline execution: + +```bash +# Parameters are stored with the base path prefix +/one-observability-workshop/database-endpoint +/one-observability-workshop/cluster-name +/one-observability-workshop/vpc-id +``` + +**Fallback Behavior:** + +- **Source**: If CodeConnection ARN is not provided, S3 bucket source is used automatically +- **Configuration**: If Parameter Store parameters are not found, local .env files are used as fallback +- **Local Development**: Always uses local .env files for immediate development iteration + +**Benefits:** + +- **Security**: CodeConnection provides secure GitHub integration without storing tokens +- **Centralized Config**: Parameter Store enables centralized configuration management +- **Flexibility**: Seamless fallback to S3/local files ensures compatibility +- **Scalability**: Parameter Store supports multiple environments and teams **Usage:** + ```bash ./scripts/deploy-check.sh ``` The script will: + - Validate AWS credentials and display current role/account - Check if the S3 bucket exists (create if needed) - Verify the repository archive exists in S3 (upload if needed) @@ -174,16 +254,18 @@ The script will: The `src/cdk/scripts/validate-account.sh` script validates account-specific configurations and sets environment variables. **Usage:** + ```bash ./src/cdk/scripts/validate-account.sh src/cdk/.env ``` The script will: + - Check if X-Ray transaction search is configured for CloudWatch Logs - When `ENABLE_PET_FOOD_AGENT=true`, retrieve and map availability zones for AgentCore deployment - Update the `.env` file with: - - `AUTO_TRANSACTION_SEARCH_CONFIGURED`: Whether X-Ray is configured - - `AVAILABILITY_ZONES`: Comma-separated list of AZ names (when AgentCore is enabled) + - `AUTO_TRANSACTION_SEARCH_CONFIGURED`: Whether X-Ray is configured + - `AVAILABILITY_ZONES`: Comma-separated list of AZ names (when AgentCore is enabled) **Note:** This script is automatically called by `deploy-check.sh` but can be run independently. @@ -192,11 +274,13 @@ The script will: The `src/cdk/scripts/redeploy-app.sh` script helps developers quickly redeploy individual microservices for testing new versions. **Prerequisites:** + - AWS CLI configured with appropriate credentials - One of: Docker, Finch, or Podman installed - Deployed One Observability Demo infrastructure **Usage:** + ```bash ./src/cdk/scripts/redeploy-app.sh ``` @@ -208,12 +292,14 @@ See [Application Redeployment Guide](docs/application-redeployment.md) for detai The `src/cdk/scripts/seed-dynamodb.sh` script helps populate DynamoDB tables with initial pet adoption data. **Prerequisites:** + - AWS CLI configured with appropriate credentials - `jq` command-line JSON processor installed - CDK resources must be deployed first - The script uses data from `src/cdk/scripts/seed.json` **Usage:** + ```bash # Interactive mode ./src/cdk/scripts/seed-dynamodb.sh @@ -223,6 +309,7 @@ The `src/cdk/scripts/seed-dynamodb.sh` script helps populate DynamoDB tables wit ``` The script will: + - Accept table name as parameter for non-interactive usage - List all DynamoDB tables in your account (interactive mode) - Automatically suggest tables containing "Petadoption" in the name (interactive mode) @@ -236,15 +323,18 @@ The script will: The `src/cdk/scripts/get-parameter.sh` script retrieves values from AWS Systems Manager Parameter Store using the configured prefix. **Prerequisites:** + - AWS CLI configured with appropriate credentials - CDK resources must be deployed first **Usage:** + ```bash ./src/cdk/scripts/get-parameter.sh ``` **Example:** + ```bash ./src/cdk/scripts/get-parameter.sh database-endpoint ``` @@ -252,13 +342,27 @@ The `src/cdk/scripts/get-parameter.sh` script retrieves values from AWS Systems This retrieves the parameter `/petstore/database-endpoint` from Parameter Store. **Return Values:** + - Parameter value if found - `-1` if parameter not found or invalid key - `-2` if access denied +## Troubleshooting + +### CDK Bootstrap Stack Deletion Issue + +When the delete step function fails, the CDK bootstrap stack may be removed before some stacks are cleaned up. Under that situation, stacks cannot be deleted because the CloudFormation execution role was removed with the CDK bootstrap stack. + +To regain access to delete the resources, bootstrap again from CloudShell by running the following commands: + +```bash +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +cdk bootstrap aws://${AWS_ACCOUNT_ID}/${AWS_REGION} --toolkit-stack-name CDKToolkitPetsite --qualifier petsite +``` + ## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. ## Licensing diff --git a/docs/codeconnection-parameter-store-integration.md b/docs/codeconnection-parameter-store-integration.md new file mode 100644 index 00000000..0a0f5c93 --- /dev/null +++ b/docs/codeconnection-parameter-store-integration.md @@ -0,0 +1,208 @@ +# CodeConnection and Parameter Store Integration + +This document describes the implementation of CodeConnection integration and Parameter Store configuration management for the One Observability Workshop CDK pipeline. + +## Overview + +The pipeline now supports two modes of operation: + +1. **CodeConnection Mode**: Uses AWS CodeConnection to connect directly to GitHub repositories +2. **S3 Fallback Mode**: Uses S3 bucket as the source (existing functionality) + +Additionally, configuration management has been moved from local `.env` files to AWS Systems Manager Parameter Store for better security and centralization. The implementation uses a **single parameter** approach for efficiency, storing the entire configuration as one parameter instead of multiple individual parameters. + +## Implementation Details + +### CloudFormation Template Updates + +**File**: `src/templates/codebuild-deployment-template.yaml` + +#### New Parameters + +- `pCodeConnectionArn`: Optional CodeConnection ARN for GitHub integration +- `pParameterStoreBasePath`: Base path in Parameter Store for configuration storage (default: `/petstore`) + +#### New CloudFormation Resources + +- `rWorkshopConfigParameter`: Parameter Store parameter created as CloudFormation resource for proper lifecycle management + +```yaml +rWorkshopConfigParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub '${pParameterStoreBasePath}/${AWS::StackName}/config' + Type: String + Value: '# Configuration will be populated by CodeBuild' + Description: !Sub 'Workshop configuration for stack ${AWS::StackName}' +``` + +#### Updated IAM Permissions + +The CodeBuild service role now includes Parameter Store permissions: + +```yaml +- PolicyName: ParameterStoreAccess + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParametersByPath + - ssm:PutParameter + - ssm:DeleteParameter + Resource: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${pParameterStoreBasePath}/*' +``` + +#### Updated BuildSpec + +The initial CodeBuild job now: + +1. Populates the CloudFormation-created Parameter Store parameter with the entire configuration +2. Handles conditional repository source setup (CodeConnection vs S3) +3. No longer stores CodeConnection ARN separately in Parameter Store (it's included in the main configuration) + +### CDK Pipeline Updates + +#### Environment Configuration + +**File**: `src/cdk/bin/environment.ts` + +Added support for CodeConnection ARN via environment variable: + +```typescript +export const CODE_CONNECTION_ARN = process.env.CODE_CONNECTION_ARN; +``` + +#### Workshop Entry Point + +**File**: `src/cdk/bin/workshop.ts` + +Updated the CDKPipeline constructor to include new parameters: + +```typescript +const pipeline = new CDKPipeline(this, 'Pipeline', { + // ... existing properties + codeConnectionArn: context.codeConnectionArn || CODE_CONNECTION_ARN, + parameterStoreBasePath: context.parameterStoreBasePath, + stackName: stackName, +}); +``` + +#### Local Development Configuration + +**File**: `src/cdk/bin/local.ts` + +Updated the ContainersStack configuration with conditional source selection: + +```typescript +if (codeConnectionArn) { + // Use CodeConnection as pipeline source + repositorySource = RepositorySource.connection({ + organizationName, + repositoryName, + branchName, + connectionArn: codeConnectionArn, + }); +} else { + // Fallback to S3 bucket source + repositorySource = RepositorySource.s3({ + configBucketName, + repositoryName, + branchName, + }); +} +``` + +#### Parameter Store Integration + +Both the synthesis step and exports dashboard step now use a reusable script to retrieve configuration from Parameter Store with the single parameter approach. + +### Configuration Retrieval Script + +**File**: `src/cdk/scripts/retrieve-config.sh` + +This reusable bash script: + +1. Accepts Parameter Store parameter name as argument +2. Retrieves the single configuration parameter containing the entire `.env` file content +3. Creates `.env` file from the Parameter Store content +4. Provides fallback to local `.env` file if Parameter Store retrieval fails + +Usage: + +```bash +./scripts/retrieve-config.sh "/petstore/stack-name/config" +``` + +The script implements the **single parameter approach** for optimal performance, retrieving all configuration in one API call instead of multiple calls for individual parameters. + +## Usage + +### Deploying with CodeConnection + +1. Create a CodeConnection in AWS Console +2. Deploy the CloudFormation template with the CodeConnection ARN: + +```bash +aws cloudformation deploy \ + --template-file src/templates/codebuild-deployment-template.yaml \ + --stack-name one-observability-pipeline \ + --parameter-overrides \ + pCodeConnectionArn=arn:aws:codeconnections:region:account:connection/connection-id \ + pParameterStoreBasePath=/oneobservability/workshop \ + --capabilities CAPABILITY_IAM +``` + +### Deploying with S3 Fallback + +Deploy without the CodeConnection ARN parameter: + +```bash +aws cloudformation deploy \ + --template-file src/templates/codebuild-deployment-template.yaml \ + --stack-name one-observability-pipeline \ + --parameter-overrides \ + pParameterStoreBasePath=/oneobservability/workshop \ + --capabilities CAPABILITY_IAM +``` + +### Parameter Store Configuration + +The configuration is automatically populated by the CloudFormation template's CodeBuild process. The **single parameter** contains the entire `.env` file content: + +```bash +# Example: View the configuration parameter created by CloudFormation +aws ssm get-parameter \ + --name "/petstore/your-stack-name/config" \ + --with-decryption + +# The parameter value contains the entire configuration as a single .env format: +# CUSTOM_ENABLE_WAF=true +# AWS_REGION=us-east-1 +# ORGANIZATION_NAME=aws-samples +# CODE_CONNECTION_ARN=arn:aws:codeconnections:region:account:connection/connection-id +# ... (all other configuration variables) +``` + +**Note**: The parameter is managed by CloudFormation and automatically populated during the initial CodeBuild process. Manual configuration changes should be made by updating the source configuration file and rerunning the pipeline. + +## Benefits + +1. **Security**: Configuration stored centrally in Parameter Store instead of local files +2. **Flexibility**: Supports both CodeConnection and S3 sources +3. **Consistency**: Same configuration retrieval mechanism across all pipeline steps +4. **Maintainability**: Reusable script reduces code duplication +5. **Auditability**: Parameter Store provides audit trails for configuration changes + +## Migration Path + +Existing deployments using S3 bucket source will continue to work unchanged. To migrate to CodeConnection: + +1. Create a CodeConnection in AWS Console +2. Update the CloudFormation stack with the CodeConnection ARN +3. Migrate configuration from `.env` files to Parameter Store +4. Redeploy the pipeline + +The pipeline will automatically detect and use the CodeConnection when available, falling back to S3 bucket source otherwise. diff --git a/src/applications/canaries/housekeeping/nodejs/package.json b/src/applications/canaries/housekeeping/nodejs/node_modules/package.json similarity index 100% rename from src/applications/canaries/housekeeping/nodejs/package.json rename to src/applications/canaries/housekeeping/nodejs/node_modules/package.json diff --git a/src/applications/canaries/petsite-canary/nodejs/index.js b/src/applications/canaries/petsite-canary/nodejs/index.js deleted file mode 100644 index 464fb5a1..00000000 --- a/src/applications/canaries/petsite-canary/nodejs/index.js +++ /dev/null @@ -1,121 +0,0 @@ -//Update to use the new petsite URL - -var synthetics = require('Synthetics'); -const log = require('SyntheticsLogger'); -const AWSXRay = require('aws-xray-sdk-core'); - -const recordedScript = async function () { - return await AWSXRay.captureAsyncFunc('canary-execution', async (subsegment) => { - let page = await synthetics.getPage(); - - const navigationPromise = page.waitForNavigation(); - - // Try to read from SSM, fallback to environment variable - let petsiteUrl = process.env.PETSITE_URL; - const ssmParameterName = process.env.PETSITE_URL_PARAMETER_NAME; - if (!ssmParameterName) { - throw new Error('SSM parameter name not set'); - } - - // Attempt to read from SSM - try { - const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm'); - const ssm = AWSXRay.captureAWSv3Client(new SSMClient({})); - const command = new GetParameterCommand({ - Name: ssmParameterName, - WithDecryption: false, - }); - const parameter = await ssm.send(command); - - if (parameter.Parameter && parameter.Parameter.Value) { - petsiteUrl = parameter.Parameter.Value; - log.info('Successfully retrieved petsite URL from SSM: ' + petsiteUrl); - } - } catch { - log.info('SSM access failed, using environment variable URL: ' + petsiteUrl); - } - - log.info('Starting canary execution with URL: ' + petsiteUrl); - log.info('SSM Parameter to monitor: ' + ssmParameterName); - - try { - await synthetics.executeStep('Goto_0', async function () { - await page.goto(petsiteUrl, { waitUntil: 'domcontentloaded', timeout: 60_000 }); - }); - - await page.setViewport({ width: 1448, height: 857 }); - - await synthetics.executeStep('Click_1', async function () { - await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor'); - await page.click('.pet-filters #Varieties_SelectedPetColor'); - }); - - await synthetics.executeStep('Select_2', async function () { - await page.select('.pet-filters #Varieties_SelectedPetColor', 'brown'); - }); - - await synthetics.executeStep('Click_3', async function () { - await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor'); - await page.click('.pet-filters #Varieties_SelectedPetColor'); - }); - - await synthetics.executeStep('Click_4', async function () { - await page.waitForSelector('.pet-wrapper #searchpets'); - await page.click('.pet-wrapper #searchpets'); - }); - - await navigationPromise; - - await synthetics.executeStep('Click_5', async function () { - await page.waitForSelector('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button'); - await page.click('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button'); - }); - - await navigationPromise; - - await synthetics.executeStep('Click_6', async function () { - await page.waitForSelector('.row > .col-md-6 > .form-group > form > .btn'); - await page.click('.row > .col-md-6 > .form-group > form > .btn'); - }); - - await navigationPromise; - - await synthetics.executeStep('Click_7', async function () { - await page.waitForSelector('.row > .col-md-6 > .pet-items > div > .btn-primary'); - await page.click('.row > .col-md-6 > .pet-items > div > .btn-primary'); - }); - - await navigationPromise; - - await synthetics.executeStep('Click_8', async function () { - await page.waitForSelector('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary'); - await page.click('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary'); - }); - - await synthetics.executeStep('Click_9', async function () { - await page.waitForSelector('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)'); - await page.click('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)'); - }); - - await navigationPromise; - - await synthetics.executeStep('Click_10', async function () { - await page.waitForSelector('.row #payCheckoutBtn'); - await page.click('.row #payCheckoutBtn'); - }); - - log.info(' canary execution completed successfully'); - if (subsegment) subsegment.close(); - } catch (error) { - log.error(' canary execution failed: ' + error.message); - if (subsegment) { - subsegment.addError(error); - subsegment.close(error); - } - throw error; - } - }); -}; -exports.handler = async () => { - return await recordedScript(); -}; diff --git a/src/applications/canaries/petsite-canary/nodejs/node_modules/index.js b/src/applications/canaries/petsite-canary/nodejs/node_modules/index.js index d7384dad..464fb5a1 100644 --- a/src/applications/canaries/petsite-canary/nodejs/node_modules/index.js +++ b/src/applications/canaries/petsite-canary/nodejs/node_modules/index.js @@ -8,118 +8,114 @@ const recordedScript = async function () { return await AWSXRay.captureAsyncFunc('canary-execution', async (subsegment) => { let page = await synthetics.getPage(); - const navigationPromise = page.waitForNavigation(); - - // Try to read from SSM, fallback to environment variable - let petsiteUrl = process.env.PETSITE_URL; - const ssmParameterName = process.env.PETSITE_URL_PARAMETER_NAME; - if (!ssmParameterName) { - throw new Error('SSM parameter name not set'); - } - - // Attempt to read from SSM - try { - const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm'); - const ssm = AWSXRay.captureAWSv3Client(new SSMClient({})); - const command = new GetParameterCommand({ - Name: ssmParameterName, - WithDecryption: false, - }); - const parameter = await ssm.send(command); - - if (parameter.Parameter && parameter.Parameter.Value) { - petsiteUrl = parameter.Parameter.Value; - log.info('Successfully retrieved petsite URL from SSM: ' + petsiteUrl); + const navigationPromise = page.waitForNavigation(); + + // Try to read from SSM, fallback to environment variable + let petsiteUrl = process.env.PETSITE_URL; + const ssmParameterName = process.env.PETSITE_URL_PARAMETER_NAME; + if (!ssmParameterName) { + throw new Error('SSM parameter name not set'); + } + + // Attempt to read from SSM + try { + const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm'); + const ssm = AWSXRay.captureAWSv3Client(new SSMClient({})); + const command = new GetParameterCommand({ + Name: ssmParameterName, + WithDecryption: false, + }); + const parameter = await ssm.send(command); + + if (parameter.Parameter && parameter.Parameter.Value) { + petsiteUrl = parameter.Parameter.Value; + log.info('Successfully retrieved petsite URL from SSM: ' + petsiteUrl); + } + } catch { + log.info('SSM access failed, using environment variable URL: ' + petsiteUrl); } - } catch (error) { - log.info('SSM access failed, using environment variable URL: ' + petsiteUrl); - } - - log.info('Starting canary execution with URL: ' + petsiteUrl); - log.info('SSM Parameter to monitor: ' + ssmParameterName); - - try { - await synthetics.executeStep('Goto_0', async function() { - await page.goto(petsiteUrl, {waitUntil: 'domcontentloaded', timeout: 60000}) - }) - - await page.setViewport({ width: 1448, height: 857 }) - - await synthetics.executeStep('Click_1', async function() { - await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor') - await page.click('.pet-filters #Varieties_SelectedPetColor') - }) - - await synthetics.executeStep('Select_2', async function() { - await page.select('.pet-filters #Varieties_SelectedPetColor', 'brown') - }) - - await synthetics.executeStep('Click_3', async function() { - await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor') - await page.click('.pet-filters #Varieties_SelectedPetColor') - }) - - await synthetics.executeStep('Click_4', async function() { - await page.waitForSelector('.pet-wrapper #searchpets') - await page.click('.pet-wrapper #searchpets') - }) - - await navigationPromise - - await synthetics.executeStep('Click_5', async function() { - await page.waitForSelector('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button') - await page.click('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button') - }) - - await navigationPromise - - await synthetics.executeStep('Click_6', async function() { - await page.waitForSelector('.row > .col-md-6 > .form-group > form > .btn') - await page.click('.row > .col-md-6 > .form-group > form > .btn') - }) - - await navigationPromise - - await synthetics.executeStep('Click_7', async function() { - await page.waitForSelector('.row > .col-md-6 > .pet-items > div > .btn-primary') - await page.click('.row > .col-md-6 > .pet-items > div > .btn-primary') - }) - - await navigationPromise - - await synthetics.executeStep('Click_8', async function() { - await page.waitForSelector('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary') - await page.click('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary') - }) - - await synthetics.executeStep('Click_9', async function() { - await page.waitForSelector('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)') - await page.click('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)') - }) - - await navigationPromise - - await synthetics.executeStep('Click_10', async function() { - await page.waitForSelector('.row #payCheckoutBtn') - await page.click('.row #payCheckoutBtn') - }) - - log.info(' canary execution completed successfully'); - if (subsegment) subsegment.close(); - } catch (error) { - log.error(' canary execution failed: ' + error.message); - if (subsegment) { - subsegment.addError(error); - subsegment.close(error); + + log.info('Starting canary execution with URL: ' + petsiteUrl); + log.info('SSM Parameter to monitor: ' + ssmParameterName); + + try { + await synthetics.executeStep('Goto_0', async function () { + await page.goto(petsiteUrl, { waitUntil: 'domcontentloaded', timeout: 60_000 }); + }); + + await page.setViewport({ width: 1448, height: 857 }); + + await synthetics.executeStep('Click_1', async function () { + await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor'); + await page.click('.pet-filters #Varieties_SelectedPetColor'); + }); + + await synthetics.executeStep('Select_2', async function () { + await page.select('.pet-filters #Varieties_SelectedPetColor', 'brown'); + }); + + await synthetics.executeStep('Click_3', async function () { + await page.waitForSelector('.pet-filters #Varieties_SelectedPetColor'); + await page.click('.pet-filters #Varieties_SelectedPetColor'); + }); + + await synthetics.executeStep('Click_4', async function () { + await page.waitForSelector('.pet-wrapper #searchpets'); + await page.click('.pet-wrapper #searchpets'); + }); + + await navigationPromise; + + await synthetics.executeStep('Click_5', async function () { + await page.waitForSelector('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button'); + await page.click('.container > .pet-items > .pet-item:nth-child(1) > form > .pet-button'); + }); + + await navigationPromise; + + await synthetics.executeStep('Click_6', async function () { + await page.waitForSelector('.row > .col-md-6 > .form-group > form > .btn'); + await page.click('.row > .col-md-6 > .form-group > form > .btn'); + }); + + await navigationPromise; + + await synthetics.executeStep('Click_7', async function () { + await page.waitForSelector('.row > .col-md-6 > .pet-items > div > .btn-primary'); + await page.click('.row > .col-md-6 > .pet-items > div > .btn-primary'); + }); + + await navigationPromise; + + await synthetics.executeStep('Click_8', async function () { + await page.waitForSelector('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary'); + await page.click('.row > .col-md-4:nth-child(1) > .card > .card-body > .btn-primary'); + }); + + await synthetics.executeStep('Click_9', async function () { + await page.waitForSelector('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)'); + await page.click('.pet-header > .container > .row > .col-lg-8 > a:nth-child(5)'); + }); + + await navigationPromise; + + await synthetics.executeStep('Click_10', async function () { + await page.waitForSelector('.row #payCheckoutBtn'); + await page.click('.row #payCheckoutBtn'); + }); + + log.info(' canary execution completed successfully'); + if (subsegment) subsegment.close(); + } catch (error) { + log.error(' canary execution failed: ' + error.message); + if (subsegment) { + subsegment.addError(error); + subsegment.close(error); + } + throw error; } - throw error; - } }); }; exports.handler = async () => { return await recordedScript(); }; - - - - diff --git a/src/applications/canaries/petsite-canary/nodejs/package.json b/src/applications/canaries/petsite-canary/nodejs/node_modules/package.json similarity index 100% rename from src/applications/canaries/petsite-canary/nodejs/package.json rename to src/applications/canaries/petsite-canary/nodejs/node_modules/package.json diff --git a/src/cdk/bin/constants.ts b/src/cdk/bin/constants.ts index f35ac296..a5ecf02d 100644 --- a/src/cdk/bin/constants.ts +++ b/src/cdk/bin/constants.ts @@ -96,6 +96,15 @@ export const CLOUDMAP_NAMESPACE_ARN_EXPORT_NAME = 'public:WorkshopCloudMapNamesp // Assets Export Names export const ASSETS_BUCKET_NAME_EXPORT_NAME = 'public:WorkshopAssetsBucketName'; export const ASSETS_BUCKET_ARN_EXPORT_NAME = 'public:WorkshopAssetsBucketArn'; +export const CLOUDFRONT_DOMAIN_EXPORT_NAME = 'public:WorkshopCloudFrontDomain'; +export const CLOUDFRONT_DISTRIBUTION_ID_EXPORT_NAME = 'public:WorkshopCloudFrontDistributionId'; + +// Application URL Export Names +export const PETSITE_URL_EXPORT_NAME = 'public:WorkshopPetSiteUrl'; +export const STATUS_UPDATER_API_URL_EXPORT_NAME = 'public:WorkshopStatusUpdaterApiUrl'; + +// Pipeline Export Names +export const PIPELINE_ARN_EXPORT_NAME = 'public:WorkshopPipelineArn'; // EventBridge Export Names export const EVENTBUS_ARN_EXPORT_NAME = 'public:WorkshopEventBusArn'; diff --git a/src/cdk/bin/environment.ts b/src/cdk/bin/environment.ts index 7b10145c..00ab2557 100644 --- a/src/cdk/bin/environment.ts +++ b/src/cdk/bin/environment.ts @@ -200,7 +200,7 @@ export const PET_IMAGES = [ ]; /** Prefix for AWS Systems Manager Parameter Store parameters */ -export const PARAMETER_STORE_PREFIX = '/petstore'; +export const PARAMETER_STORE_PREFIX = process.env.PARAMETER_STORE_BASE_PATH || '/petstore'; /** Lambda function configuration for pet status updater */ export const STATUS_UPDATER_FUNCTION = { @@ -319,3 +319,7 @@ export const ENABLE_PET_FOOD_AGENT = process.env.ENABLE_PET_FOOD_AGENT == 'true' export const AVAILABILITY_ZONES = process.env.AVAILABILITY_ZONES?.split(',') || undefined; /** Enables OpenSearch Application creation */ export const ENABLE_OPENSEARCH_APPLICATION = process.env.ENABLE_OPENSEARCH_APPLICATION == 'true' || false; + +/** CodeConnection ARN for GitHub integration (optional) */ +export const CODE_CONNECTION_ARN = process.env.CODE_CONNECTION_ARN || undefined; +export const CONFIG_PARAM_NAME = process.env.CONFIG_PARAM_NAME || undefined; diff --git a/src/cdk/bin/local.ts b/src/cdk/bin/local.ts index d3eec119..e6862178 100644 --- a/src/cdk/bin/local.ts +++ b/src/cdk/bin/local.ts @@ -33,6 +33,7 @@ import { MICROSERVICES_PLACEMENT, PET_IMAGES, TAGS, + CODE_CONNECTION_ARN, } from './environment'; import { ContainersStack } from '../lib/stages/containers'; import { AwsSolutionsChecks } from 'cdk-nag'; @@ -80,8 +81,8 @@ if (CUSTOM_ENABLE_WAF && process.env?.AWS_REGION != 'us-east-1') { /** Validate required environment variables */ const s3BucketName = process.env.CONFIG_BUCKET; -if (!s3BucketName) { - throw new Error('CONFIG_BUCKET environment variable is not set'); +if (!s3BucketName && !CODE_CONNECTION_ARN) { + throw new Error('CONFIG_BUCKET or CODE_CONNECTION_ARN environment variable must be set'); } const branch_name = process.env.BRANCH_NAME; @@ -91,10 +92,22 @@ if (!branch_name) { /** Deploy container applications stack with ECS and EKS services */ const containers = new ContainersStack(app, 'DevApplicationsStack', { - source: { - bucketName: s3BucketName, - bucketKey: `repo/refs/heads/${branch_name}/repo.zip`, - }, + // Conditionally use CodeConnection or S3 source based on environment + ...(CODE_CONNECTION_ARN + ? { + codeConnectionSource: { + connectionArn: CODE_CONNECTION_ARN, + organizationName: process.env.ORGANIZATION_NAME || 'aws-samples', + repositoryName: process.env.REPOSITORY_NAME || 'one-observability-demo', + branchName: branch_name, + }, + } + : { + source: { + bucketName: s3BucketName!, + bucketKey: `repo/refs/heads/${branch_name}/repo.zip`, + }, + }), tags: TAGS, applicationList: APPLICATION_LIST, env: { diff --git a/src/cdk/bin/workshop.ts b/src/cdk/bin/workshop.ts index cfca70a8..5cacdc5a 100644 --- a/src/cdk/bin/workshop.ts +++ b/src/cdk/bin/workshop.ts @@ -43,6 +43,8 @@ import { MICROSERVICES_PLACEMENT, LAMBDA_FUNCTIONS, CANARY_FUNCTIONS, + CODE_CONNECTION_ARN, + CONFIG_PARAM_NAME, } from './environment'; /** Main CDK application instance */ @@ -53,11 +55,13 @@ const app = new App(); * Configuration values are resolved from CDK context first, then fall back to environment variables. */ new CDKPipeline(app, 'OneObservability', { - configBucketName: app.node.tryGetContext('configBucketName') || CONFIG_BUCKET, - branchName: app.node.tryGetContext('branchName') || BRANCH_NAME, - organizationName: app.node.tryGetContext('organizationName') || ORGANIZATION_NAME, - repositoryName: app.node.tryGetContext('repositoryName') || REPOSITORY_NAME, - workingFolder: app.node.tryGetContext('workingFolder') || WORKING_FOLDER, + configBucketName: CONFIG_BUCKET, + branchName: BRANCH_NAME, + organizationName: ORGANIZATION_NAME, + repositoryName: REPOSITORY_NAME, + workingFolder: WORKING_FOLDER, + codeConnectionArn: CODE_CONNECTION_ARN, + configurationParameterName: CONFIG_PARAM_NAME, env: { account: ACCOUNT_ID, region: REGION, diff --git a/src/cdk/lib/constructs/assets.ts b/src/cdk/lib/constructs/assets.ts index 97b7e1d8..eaf274cd 100644 --- a/src/cdk/lib/constructs/assets.ts +++ b/src/cdk/lib/constructs/assets.ts @@ -20,6 +20,8 @@ import { PARAMETER_STORE_PREFIX } from '../../bin/environment'; import { ASSETS_BUCKET_NAME_EXPORT_NAME, ASSETS_BUCKET_ARN_EXPORT_NAME, + CLOUDFRONT_DOMAIN_EXPORT_NAME, + CLOUDFRONT_DISTRIBUTION_ID_EXPORT_NAME, SSM_PARAMETER_NAMES, } from '../../bin/constants'; @@ -208,13 +210,13 @@ export class WorkshopAssets extends Construct { // CloudFront distribution outputs new CfnOutput(this, 'CloudFrontDomainOutput', { value: this.distribution.distributionDomainName, - exportName: 'WorkshopCloudFrontDomain', + exportName: CLOUDFRONT_DOMAIN_EXPORT_NAME, description: 'Workshop CloudFront Distribution Domain Name', }); new CfnOutput(this, 'CloudFrontDistributionIdOutput', { value: this.distribution.distributionId, - exportName: 'WorkshopCloudFrontDistributionId', + exportName: CLOUDFRONT_DISTRIBUTION_ID_EXPORT_NAME, description: 'Workshop CloudFront Distribution ID', }); diff --git a/src/cdk/lib/constructs/opensearch-pipeline.ts b/src/cdk/lib/constructs/opensearch-pipeline.ts index dd8009d4..58efac85 100644 --- a/src/cdk/lib/constructs/opensearch-pipeline.ts +++ b/src/cdk/lib/constructs/opensearch-pipeline.ts @@ -15,6 +15,7 @@ import { import { Utilities } from '../utils/utilities'; import { PARAMETER_STORE_PREFIX } from '../../bin/environment'; import { OpenSearchCollection } from './opensearch-collection'; +import { NagSuppressions } from 'cdk-nag'; /** * Properties for configuring OpenSearchPipeline construct @@ -123,6 +124,14 @@ export class OpenSearchPipeline extends Construct { ? properties.openSearchCollection.collectionArn : properties.openSearchCollection.collection.attrArn; + // Create CloudWatch log group for pipeline logs + // OpenSearch Ingestion requires log groups to use /aws/vendedlogs/ prefix + const logGroup = new LogGroup(this, 'PipelineLogGroup', { + retention: RetentionDays.ONE_WEEK, + removalPolicy: RemovalPolicy.DESTROY, + logGroupName: `/aws/vendedlogs/opensearch-ingestion/${pipelineName}`, + }); + // Create IAM role for the pipeline this.pipelineRole = new Role(this, 'PipelineRole', { assumedBy: new ServicePrincipal('osis-pipelines.amazonaws.com'), @@ -163,13 +172,6 @@ export class OpenSearchPipeline extends Construct { }), ); - // Create CloudWatch log group for pipeline logs - // OpenSearch Ingestion requires log groups to use /aws/vendedlogs/ prefix - const logGroup = new LogGroup(this, 'PipelineLogGroup', { - retention: RetentionDays.ONE_WEEK, - removalPolicy: RemovalPolicy.DESTROY, - }); - // Generate pipeline configuration YAML const pipelineConfiguration = this.generatePipelineConfiguration( collectionEndpoint, @@ -188,7 +190,7 @@ export class OpenSearchPipeline extends Construct { logPublishingOptions: { isLoggingEnabled: true, cloudWatchLogDestination: { - logGroup: `/aws/vendedlogs/opensearch-ingestion/${pipelineName}`, + logGroup: logGroup.logGroupName, }, }, // Add tags for resource management @@ -208,6 +210,13 @@ export class OpenSearchPipeline extends Construct { ], }); + NagSuppressions.addResourceSuppressions(logGroup, [ + { + id: 'Workshop-CWL3', + reason: 'OpenSearch pipeline log group name must include vendedlogs or creation will fail', + }, + ]); + // Add dependencies to ensure resources are created in correct order this.pipeline.node.addDependency(this.pipelineRole); this.pipeline.node.addDependency(logGroup); diff --git a/src/cdk/lib/microservices/petsite.ts b/src/cdk/lib/microservices/petsite.ts index 8af7049d..210d325a 100644 --- a/src/cdk/lib/microservices/petsite.ts +++ b/src/cdk/lib/microservices/petsite.ts @@ -29,7 +29,7 @@ import { LoadBalancerV2Origin } from 'aws-cdk-lib/aws-cloudfront-origins'; import { NagSuppressions } from 'cdk-nag'; import { Utilities } from '../utils/utilities'; import { DEFAULT_RETENTION_DAYS, PARAMETER_STORE_PREFIX } from '../../bin/environment'; -import { SSM_PARAMETER_NAMES } from '../../bin/constants'; +import { SSM_PARAMETER_NAMES, PETSITE_URL_EXPORT_NAME } from '../../bin/constants'; import { Peer, Port, PrefixList } from 'aws-cdk-lib/aws-ec2'; import { Bucket, ObjectOwnership } from 'aws-cdk-lib/aws-s3'; import { CfnOutput, Duration, RemovalPolicy, Stack } from 'aws-cdk-lib'; @@ -296,7 +296,7 @@ export class PetSite extends EKSDeployment { createOutputs(): void { new CfnOutput(this, 'PetSiteUrl', { value: `https://${this.distribution.distributionDomainName}`, - exportName: 'PetSiteUrl', + exportName: PETSITE_URL_EXPORT_NAME, description: 'The URL of the PetSite application', }); diff --git a/src/cdk/lib/pipeline.ts b/src/cdk/lib/pipeline.ts index ef25b0e2..bae52eb5 100644 --- a/src/cdk/lib/pipeline.ts +++ b/src/cdk/lib/pipeline.ts @@ -16,7 +16,7 @@ import { CfnOutput, RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; import { BuildSpec, LinuxBuildImage } from 'aws-cdk-lib/aws-codebuild'; import { PipelineType } from 'aws-cdk-lib/aws-codepipeline'; import { IRole, Policy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; -import { BlockPublicAccess, Bucket, BucketEncryption } from 'aws-cdk-lib/aws-s3'; +import { BlockPublicAccess, Bucket, BucketEncryption, IBucket } from 'aws-cdk-lib/aws-s3'; import { CodeBuildStep, CodePipeline, CodePipelineSource } from 'aws-cdk-lib/pipelines'; import { NagSuppressions } from 'cdk-nag'; import { Construct } from 'constructs'; @@ -31,6 +31,7 @@ import { ComputeStage } from './stages/compute'; import { MicroservicesStage, MicroserviceApplicationsProperties } from './stages/applications'; import { CUSTOM_ENABLE_WAF } from '../bin/environment'; import { GlobalWaf } from './constructs/waf'; +import { PIPELINE_ARN_EXPORT_NAME } from '../bin/constants'; /** * Properties for configuring the CDK Pipeline stack. @@ -39,8 +40,8 @@ import { GlobalWaf } from './constructs/waf'; * for deploying the One Observability Workshop infrastructure pipeline. */ export interface CDKPipelineProperties extends StackProps { - /** S3 bucket name containing the source code repository */ - configBucketName: string; + /** S3 bucket name containing the source code repository (required only when not using CodeConnection) */ + configBucketName?: string; /** Git branch name to deploy from */ branchName: string; /** Organization name for resource naming */ @@ -49,6 +50,10 @@ export interface CDKPipelineProperties extends StackProps { repositoryName: string; /** Working folder path within the repository */ workingFolder: string; + /** Optional CodeConnection ARN for GitHub integration. If provided, will be used instead of S3 as pipeline source. */ + codeConnectionArn?: string; + /** Base path in Parameter Store for configuration storage */ + configurationParameterName?: string; /** Optional tags to apply to all resources */ tags?: { [key: string]: string }; /** Optional properties for the core infrastructure stage */ @@ -95,14 +100,34 @@ export class CDKPipeline extends Stack { constructor(scope: Construct, id: string, properties: CDKPipelineProperties) { super(scope, id, properties); - // Create a CodePipeline source using the Specified S3 Bucket - const configBucket = Bucket.fromBucketName(this, 'ConfigBucket', properties.configBucketName); - const bucketKey = `repo/refs/heads/${properties.branchName}/repo.zip`; + // Validate required properties based on source type + if (!properties.codeConnectionArn && !properties.configBucketName) { + throw new Error('Either codeConnectionArn or configBucketName must be provided'); + } - // Use the configuration file as the pipeline trigger - const bucketSource = CodePipelineSource.s3(configBucket, bucketKey, { - trigger: S3Trigger.POLL, - }); + // Determine pipeline source based on CodeConnection availability + let pipelineSource: CodePipelineSource; + let configBucket: IBucket | undefined; + let bucketKey: string | undefined; + + if (properties.codeConnectionArn) { + // Use CodeConnection as pipeline source + pipelineSource = CodePipelineSource.connection( + `${properties.organizationName}/${properties.repositoryName}`, + properties.branchName, + { + connectionArn: properties.codeConnectionArn, + }, + ); + } else { + // Fallback to S3 bucket source + configBucket = Bucket.fromBucketName(this, 'ConfigBucket', properties.configBucketName!); + bucketKey = `repo/refs/heads/${properties.branchName}/repo.zip`; + + pipelineSource = CodePipelineSource.s3(configBucket, bucketKey, { + trigger: S3Trigger.POLL, + }); + } /** * Create an S3 bucket to store the pipeline artifacts. * The bucket has encryption at rest using a CMK and enforces encryption in transit. @@ -136,11 +161,14 @@ export class CDKPipeline extends Stack { /** * Grant access to the source bucket for the pipeline role. + * Only grant access if using S3 bucket source (not CodeConnection). */ - configBucket.grantRead(this.pipelineRole); + if (configBucket) { + configBucket.grantRead(this.pipelineRole); + } const synthStep = new CodeBuildStep('Synth', { - input: bucketSource, + input: pipelineSource, primaryOutputDirectory: `${properties.workingFolder}/cdk.out`, installCommands: ['npm i -g aws-cdk'], // Using globally installed CDK due to this issue https://github.com/aws/aws-cdk/issues/28519 @@ -149,8 +177,15 @@ export class CDKPipeline extends Stack { 'npm ci', 'npm run build', 'echo ----------------------------', - 'echo "Working with configuration:"', - 'cat .env', + 'echo "Retrieving configuration..."', + //'export LOG_LEVEL=DEBUG', + // Use the reusable script to retrieve configuration + ...(properties.configurationParameterName + ? [`./scripts/retrieve-config.sh "${properties.configurationParameterName}"`] + : [ + 'echo "Using local .env file (Parameter Store base path not configured)"', + 'cat .env || echo "No .env file found"', + ]), 'echo ----------------------------', 'cdk synth --all', ], @@ -219,10 +254,21 @@ export class CDKPipeline extends Stack { new ContainersPipelineStage(this, 'Applications', { applicationList: properties.applicationList, tags: applicationsStageTags, - source: { - bucketName: properties.configBucketName, - bucketKey: bucketKey, - }, + ...(properties.codeConnectionArn + ? { + codeConnectionSource: { + connectionArn: properties.codeConnectionArn, + organizationName: properties.organizationName, + repositoryName: properties.repositoryName, + branchName: properties.branchName, + }, + } + : { + source: { + bucketName: properties.configBucketName!, + bucketKey: bucketKey || `repo/refs/heads/${properties.branchName}/repo.zip`, + }, + }), env: properties.env, }), ); @@ -246,7 +292,10 @@ export class CDKPipeline extends Stack { }); backendWave.addStage(storageStage, { - post: [storageStage.getDDBSeedingStep(this, configBucket), storageStage.getRDSSeedingStep(this)], + post: [ + ...(configBucket ? [storageStage.getDDBSeedingStep(this, configBucket as Bucket)] : []), + storageStage.getRDSSeedingStep(this), + ], }); const computeStage = new ComputeStage(this, 'Compute', { @@ -279,12 +328,58 @@ export class CDKPipeline extends Stack { */ const exportsDashboardWave = pipeline.addWave('ExportsDashboard'); + const exportDashboardRole = new Role(this, 'ExportsDashboardRole', { + assumedBy: new ServicePrincipal('codebuild.amazonaws.com'), + description: 'CodeBuild role for exports dashboard generation', + }); + + exportDashboardRole.addToPolicy( + new PolicyStatement({ + actions: [ + 'cloudformation:DescribeStacks', + 'cloudformation:ListResources', + 'cloudformation:ListExports', + ], + resources: ['*'], + }), + ); + + exportDashboardRole.addToPolicy( + new PolicyStatement({ + actions: ['ssm:GetParameter', 'ssm:GetParameters', 'ssm:GetParametersByPath'], + resources: [ + `arn:aws:ssm:${this.region}:${this.account}:parameter${properties.configurationParameterName}*`, + ], + }), + ); + + // TODO: Broad access is needed since the bucket name is generated by CDK + exportDashboardRole.addToPolicy( + new PolicyStatement({ + actions: ['s3:PutObject'], + resources: [`arn:aws:s3:::*/*`], + }), + ); + const exportsDashboardStep = new CodeBuildStep('GenerateExportsDashboard', { - input: bucketSource, + input: pipelineSource, commands: [ `cd ${properties.workingFolder}`, + 'echo "Retrieving configuration for exports dashboard..."', + //'export LOG_LEVEL=DEBUG', + // Use the reusable script to retrieve configuration + ...(properties.configurationParameterName + ? [`./scripts/retrieve-config.sh "${properties.configurationParameterName}"`] + : [ + 'echo "Using local .env file (Parameter Store base path not configured)"', + 'cat .env || echo "No .env file found"', + ]), + // Source the .env file to make variables available as environment variables + 'echo "Environment variables in use:"', + 'cat .env', + 'set -a && source .env && set +a', 'echo "Installing Python dependencies for exports generation..."', - 'pip3 install -r /scripts/requirements.txt', + 'pip3 install -r scripts/requirements.txt', 'echo "Generating CDK exports dashboard..."', 'python3 scripts/manage-exports.py generate-dashboard', 'echo "Exports dashboard generation completed"', @@ -296,11 +391,14 @@ export class CDKPipeline extends Stack { NODE_VERSION: { value: '22.x', }, - CUSTOM_ENABLE_WAF: { - value: CUSTOM_ENABLE_WAF ? 'true' : 'false', - }, }, }, + partialBuildSpec: BuildSpec.fromObject({ + env: { + shell: 'bash', + }, + }), + role: exportDashboardRole, }); exportsDashboardWave.addPost(exportsDashboardStep); @@ -313,22 +411,36 @@ export class CDKPipeline extends Stack { pipeline.buildPipeline(); /** - * Grant access to describe Prefix lists + * Grant access to describe Prefix lists and Parameter Store */ if (pipeline.synthProject.role) { - new Policy(this, 'CloudFormationPolicy', { - statements: [ + const policyStatements = [ + new PolicyStatement({ + actions: [ + 'cloudformation:DescribeStacks', + 'cloudformation:ListResources', + 'ec2:DescribeManagedPrefixLists', + 'ec2:GetManagedPrefixListEntries', + 'ec2:DescribeAvailabilityZones', + ], + resources: ['*'], + }), + ]; + + // Add Parameter Store permissions if base path is provided + if (properties.configurationParameterName) { + policyStatements.push( new PolicyStatement({ - actions: [ - 'cloudformation:DescribeStacks', - 'cloudformation:ListResources', - 'ec2:DescribeManagedPrefixLists', - 'ec2:GetManagedPrefixListEntries', - 'ec2:DescribeAvailabilityZones', + actions: ['ssm:GetParameter', 'ssm:GetParameters', 'ssm:GetParametersByPath'], + resources: [ + `arn:aws:ssm:${this.region}:${this.account}:parameter${properties.configurationParameterName}*`, ], - resources: ['*'], }), - ], + ); + } + + new Policy(this, 'CloudFormationPolicy', { + statements: policyStatements, roles: [pipeline.synthProject.role], }); } @@ -429,7 +541,7 @@ export class CDKPipeline extends Stack { */ new CfnOutput(this, 'PipelineArn', { value: pipeline.pipeline.pipelineArn, - exportName: 'PipelineArn', + exportName: PIPELINE_ARN_EXPORT_NAME, description: 'ARN of the CI/CD pipeline for deploying workshop infrastructure', }); diff --git a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts index 3d9c6579..04863ce2 100644 --- a/src/cdk/lib/serverless/functions/status-updater/status-updater.ts +++ b/src/cdk/lib/serverless/functions/status-updater/status-updater.ts @@ -20,7 +20,7 @@ import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { IVpcEndpoint } from 'aws-cdk-lib/aws-ec2'; import { Utilities } from '../../../utils/utilities'; import { PARAMETER_STORE_PREFIX } from '../../../../bin/environment'; -import { SSM_PARAMETER_NAMES } from '../../../../bin/constants'; +import { SSM_PARAMETER_NAMES, STATUS_UPDATER_API_URL_EXPORT_NAME } from '../../../../bin/constants'; export interface StatusUpdaterServiceProperties extends WorkshopLambdaFunctionProperties { table: ITable; @@ -152,7 +152,7 @@ export class StatusUpdatedService extends WokshopLambdaFunction { new CfnOutput(this, 'StatusUpdaterApiUrl', { value: this.api.url, - exportName: 'StatusUpdaterApiUrl', + exportName: STATUS_UPDATER_API_URL_EXPORT_NAME, description: 'API Gateway URL for the pet status updater service', }); } else { diff --git a/src/cdk/lib/stages/containers.ts b/src/cdk/lib/stages/containers.ts index ba89830a..8a6ca470 100644 --- a/src/cdk/lib/stages/containers.ts +++ b/src/cdk/lib/stages/containers.ts @@ -9,6 +9,7 @@ import { Construct } from 'constructs'; import { NagSuppressions } from 'cdk-nag'; import { CodeBuildAction, + CodeStarConnectionsSourceAction, EcrBuildAndPublishAction, RegistryType, S3SourceAction, @@ -118,12 +119,28 @@ export interface S3SourceProperties { bucketKey: string; } +/** + * Properties for CodeConnection source configuration + */ +export interface CodeConnectionSourceProperties { + /** CodeConnection ARN for GitHub integration */ + connectionArn: string; + /** Organization/owner name */ + organizationName: string; + /** Repository name */ + repositoryName: string; + /** Branch name */ + branchName: string; +} + /** * Properties for the Containers Pipeline Stage */ export interface ContainersPipelineStageProperties extends StackProps { - /** S3 source configuration */ - source: S3SourceProperties; + /** S3 source configuration (used when CodeConnection is not available) */ + source?: S3SourceProperties; + /** CodeConnection source configuration */ + codeConnectionSource?: CodeConnectionSourceProperties; /** List of applications to build and deploy */ applicationList: ContainerDefinition[]; } @@ -163,8 +180,12 @@ export class ContainersStack extends Stack { constructor(scope: Construct, id: string, properties?: ContainersPipelineStageProperties) { super(scope, id, properties); - if (!properties?.source || !properties?.applicationList) { - throw new Error('Source and applicationList are required'); + if (!properties?.applicationList) { + throw new Error('ApplicationList is required'); + } + + if (!properties?.source && !properties?.codeConnectionSource) { + throw new Error('Either S3 source or CodeConnection source is required'); } const pipelineRole = new Role(this, 'PipelineRole', { @@ -214,7 +235,6 @@ export class ContainersStack extends Stack { }); const sourceOutput = new Artifact(); - const sourceBucket = Bucket.fromBucketName(this, 'SourceBucket', properties.source.bucketName); const pipelineLogArn = Arn.format( { @@ -239,19 +259,38 @@ export class ContainersStack extends Stack { roles: [pipelineRole, codeBuildRole], }); - sourceBucket.grantRead(pipelineRole); + // Determine source action based on available configuration + let sourceAction; + + if (properties.codeConnectionSource) { + // Use CodeConnection as source + sourceAction = new CodeStarConnectionsSourceAction({ + actionName: 'Source', + owner: properties.codeConnectionSource.organizationName, + repo: properties.codeConnectionSource.repositoryName, + branch: properties.codeConnectionSource.branchName, + connectionArn: properties.codeConnectionSource.connectionArn, + output: sourceOutput, + }); + } else if (properties.source) { + // Fallback to S3 source + const sourceBucket = Bucket.fromBucketName(this, 'SourceBucket', properties.source.bucketName); + sourceBucket.grantRead(pipelineRole); + + sourceAction = new S3SourceAction({ + actionName: 'Source', + bucket: sourceBucket, + bucketKey: properties.source.bucketKey, + output: sourceOutput, + trigger: S3Trigger.POLL, + }); + } else { + throw new Error('No valid source configuration provided'); + } // Ensure CloudWatch policy is attached before pipeline actions this.pipeline.node.addDependency(cloudWatchPolicy); - const sourceAction = new S3SourceAction({ - actionName: 'Source', - bucket: sourceBucket, - bucketKey: properties.source.bucketKey, - output: sourceOutput, - trigger: S3Trigger.POLL, - }); - this.pipeline.addStage({ stageName: 'Source', actions: [sourceAction], diff --git a/src/cdk/scripts/manage-exports.py b/src/cdk/scripts/manage-exports.py index 0a5f6217..ad7f7c78 100755 --- a/src/cdk/scripts/manage-exports.py +++ b/src/cdk/scripts/manage-exports.py @@ -19,16 +19,76 @@ import os import sys import logging +import time from datetime import datetime, timezone from typing import Dict, List, Optional -from botocore.exceptions import ClientError, NoCredentialsError +from botocore.exceptions import ClientError, NoCredentialsError, BotoCoreError from jinja2 import Environment, FileSystemLoader, select_autoescape -# Configure logging first -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", -) + +# Enhanced logging configuration +class ColoredFormatter(logging.Formatter): + """Custom formatter to add colors to log levels""" + + COLORS = { + "DEBUG": "\033[36m", # Cyan + "INFO": "\033[32m", # Green + "WARNING": "\033[33m", # Yellow + "ERROR": "\033[31m", # Red + "CRITICAL": "\033[35m", # Magenta + } + RESET = "\033[0m" + + def format(self, record): + if hasattr(record, "levelname"): + color = self.COLORS.get(record.levelname, self.RESET) + record.levelname = f"{color}{record.levelname}{self.RESET}" + return super().format(record) + + +# Configure enhanced logging +def setup_logging(log_level: str = "INFO", enable_colors: bool = True): + """Setup enhanced logging with colors and detailed formatting""" + + # Convert string level to logging constant + numeric_level = getattr(logging, log_level.upper(), logging.INFO) + + # Create formatter + formatter: logging.Formatter + if enable_colors and hasattr(sys.stderr, "isatty") and sys.stderr.isatty(): + formatter = ColoredFormatter( + "%(asctime)s - %(levelname)s - " + "[%(name)s:%(funcName)s:%(lineno)d] - %(message)s", + ) + else: + formatter = logging.Formatter( + "%(asctime)s - %(levelname)s - " + "[%(name)s:%(funcName)s:%(lineno)d] - %(message)s", + ) + + # Setup root logger + root_logger = logging.getLogger() + root_logger.setLevel(numeric_level) + + # Clear existing handlers + for handler in root_logger.handlers[:]: + root_logger.removeHandler(handler) + + # Add console handler + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setFormatter(formatter) + root_logger.addHandler(console_handler) + + # Reduce boto3/botocore noise unless in debug mode + if numeric_level > logging.DEBUG: + logging.getLogger("boto3").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + +# Initialize logging +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO").upper() +setup_logging(LOG_LEVEL) logger = logging.getLogger(__name__) # Load environment variables from .env file (like the CDK code does) @@ -64,122 +124,412 @@ def __init__(self): self.primary_region = os.environ.get("AWS_REGION", "us-east-1") self.account_id = None self.exports_data = [] + self.session = None + + # Validate AWS environment during initialization + self._validate_aws_environment() + + def _validate_aws_environment(self) -> None: + """Validate AWS credentials and basic connectivity.""" + try: + logger.debug("Validating AWS environment...") + + # Create session to test credentials + self.session = boto3.Session() + + # Test credentials by getting caller identity + sts_client = self.session.client("sts", region_name=self.primary_region) + identity = sts_client.get_caller_identity() + + self.account_id = identity["Account"] + user_arn = identity.get("Arn", "unknown") + + logger.info("AWS authentication successful") + logger.debug(f"Account ID: {self.account_id}") + logger.debug(f"User/Role ARN: {user_arn}") + logger.debug(f"Primary region: {self.primary_region}") + + except NoCredentialsError: + logger.error("AWS credentials not found") + logger.error("Please configure credentials using:") + logger.error(" - AWS CLI: aws configure") + logger.error( + " - Environment variables: " + "AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY", + ) + logger.error(" - IAM roles (for EC2/ECS/Lambda)") + logger.error(" - AWS SSO: aws sso login") + raise + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + if error_code in ["UnauthorizedOperation", "AccessDenied"]: + logger.error("Access denied with current AWS credentials") + logger.error( + "Required permissions: sts:GetCallerIdentity, " + "cloudformation:ListExports", + ) + else: + logger.error(f"AWS API error during validation: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error validating AWS environment: {e}") + raise def get_target_regions(self) -> List[str]: """Determine which regions to scan for exports based on WAF configuration.""" + logger.debug("Determining target regions for export scanning...") + regions = [self.primary_region] # Check if WAF is enabled by looking for the environment variable - # or by checking for WAF-related exports in the current region waf_enabled = os.environ.get("CUSTOM_ENABLE_WAF", "").lower() == "true" + logger.debug(f"WAF enabled from environment: {waf_enabled}") if not waf_enabled: - # Check for WAF exports in the current region + # Check for WAF exports in the current region as fallback + logger.debug("Checking for WAF-related exports in primary region...") try: - cf_client = boto3.client( + cf_client = self.session.client( "cloudformation", region_name=self.primary_region, ) - exports = cf_client.list_exports().get("Exports", []) - waf_enabled = any("WAF" in export.get("Name", "") for export in exports) + + # Quick check for WAF exports without full enumeration + exports_found = 0 + paginator = cf_client.get_paginator("list_exports") + + for page in paginator.paginate(): + for export in page.get("Exports", []): + exports_found += 1 + export_name = export.get("Name", "") + if "WAF" in export_name.upper(): + waf_enabled = True + logger.debug(f"Found WAF export: {export_name}") + break + if waf_enabled: + break + + logger.debug( + f"Checked {exports_found} exports in {self.primary_region}", + ) + except Exception as e: - logger.warning(f"Could not check for WAF exports: {e}") + logger.warning( + "Could not check for WAF exports in %s: %s", + self.primary_region, + e, + ) if waf_enabled and "us-east-1" not in regions: regions.append("us-east-1") logger.info("WAF detected, including us-east-1 in export scan") - logger.info(f"Scanning regions: {regions}") + logger.info(f"Target regions for scanning: {regions}") return regions def extract_exports( self, filter_prefix: Optional[str] = None, exclude_internal: bool = True, + max_retries: int = 3, + retry_delay: float = 1.0, ) -> List[Dict]: """ - Extract CloudFormation exports from all target regions. + Extract CloudFormation exports from all target regions with + comprehensive error handling. Args: filter_prefix: Only include exports with names starting with this prefix exclude_internal: Exclude AWS internal exports (AWS::, CDK::, etc.) + max_retries: Maximum number of retries for failed API calls + retry_delay: Delay between retries in seconds Returns: List of export dictionaries with metadata """ - all_exports = [] - regions = self.get_target_regions() - - for region in regions: - logger.info(f"Extracting exports from region: {region}") + logger.info("=" * 60) + logger.info("STARTING EXPORT EXTRACTION") + logger.info("=" * 60) - try: - cf_client = boto3.client("cloudformation", region_name=region) - - # Get account ID from STS if not already retrieved - if not self.account_id: - sts_client = boto3.client("sts", region_name=region) - self.account_id = sts_client.get_caller_identity()["Account"] - - # Paginate through exports - paginator = cf_client.get_paginator("list_exports") - - for page in paginator.paginate(): - for export in page.get("Exports", []): - export_name = export.get("Name", "") - export_value = export.get("Value", "") - exporting_stack_id = export.get("ExportingStackId", "") + if filter_prefix: + logger.info(f"Filter prefix: '{filter_prefix}'") + else: + logger.info("No filter prefix - extracting all exports") - # Extract stack name from stack ID - stack_name = self._extract_stack_name(exporting_stack_id) + logger.info(f"Exclude internal: {exclude_internal}") + logger.info(f"Max retries: {max_retries}") - # Apply filters - if filter_prefix and not export_name.startswith(filter_prefix): - continue - - if exclude_internal and self._is_internal_export(export_name): + all_exports: List[Dict] = [] + regions = self.get_target_regions() + extraction_stats: Dict[str, int] = { + "regions_scanned": 0, + "regions_failed": 0, + "total_raw_exports": 0, + "filtered_exports": 0, + "internal_excluded": 0, + } + errors_encountered: List[str] = [] + + for region_idx, region in enumerate(regions, 1): + logger.info(f"[{region_idx}/{len(regions)}] Processing region: {region}") + extraction_stats["regions_scanned"] += 1 + + region_exports = [] + region_errors = [] + + for attempt in range(max_retries): + try: + cf_client = self.session.client( + "cloudformation", + region_name=region, + ) + + logger.debug( + "Attempt %d/%d for region %s", + attempt + 1, + max_retries, + region, + ) + + # Use paginator to handle large result sets + paginator = cf_client.get_paginator("list_exports") + page_count = 0 + + logger.debug("Starting paginated export retrieval...") + + for page in paginator.paginate(): + page_count += 1 + page_exports = page.get("Exports", []) + + logger.debug( + "Processing page %d with %d exports", + page_count, + len(page_exports), + ) + extraction_stats["total_raw_exports"] += len(page_exports) + + for export_idx, export in enumerate(page_exports): + try: + export_name = export.get("Name", "") + export_value = export.get("Value", "") + exporting_stack_id = export.get("ExportingStackId", "") + + if not export_name: + logger.warning( + "Export missing name in %s: %s", + region, + export, + ) + continue + + logger.debug(f"Processing export: {export_name}") + + # Extract stack name from stack ID + stack_name = self._extract_stack_name( + exporting_stack_id, + ) + + # Apply filters with detailed logging + if filter_prefix and not export_name.startswith( + filter_prefix, + ): + logger.debug( + "Filtered out (prefix): %s", + export_name, + ) + extraction_stats["filtered_exports"] += 1 + continue + + if exclude_internal and self._is_internal_export( + export_name, + ): + logger.debug( + "Filtered out (internal): %s", + export_name, + ) + extraction_stats["internal_excluded"] += 1 + continue + + # Get additional stack metadata with error handling + stack_info = self._get_stack_info_safe( + cf_client, + stack_name, + region, + ) + + export_data = { + "exportName": export_name, + "exportValue": export_value, + "stackName": stack_name, + "stackId": exporting_stack_id, + "region": region, + "description": stack_info.get("description", ""), + "stackStatus": stack_info.get("status", ""), + "creationTime": stack_info.get("creation_time", ""), + "tags": stack_info.get("tags", {}), + "category": self._categorize_export(export_name), + "isUrl": self._is_url_value(export_value), + "consoleUrl": self._get_console_url( + region, + stack_name, + export_name, + export_value, + ), + } + + region_exports.append(export_data) + logger.debug( + "Added export: %s -> %s", + export_name, + export_data["category"], + ) + + except Exception as export_error: + error_msg = ( + f"Error processing individual export in " + f"{region}: {export_error}" + ) + logger.error(error_msg) + region_errors.append(error_msg) + errors_encountered.append(error_msg) + continue + + logger.info( + "Successfully extracted %d exports from %s (%d pages)", + len(region_exports), + region, + page_count, + ) + all_exports.extend(region_exports) + break # Success, no need to retry + + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + error_msg = ( + f"AWS API error in {region} (attempt {attempt + 1}): " + f"{error_code} - {str(e)}" + ) + + if error_code in ["UnauthorizedOperation", "AccessDenied"]: + logger.error( + "Access denied to CloudFormation exports in %s", + region, + ) + logger.error( + "Required permissions: cloudformation:ListExports, " + "cloudformation:DescribeStacks", + ) + break # Don't retry auth errors + elif error_code == "Throttling": + logger.warning( + "API throttling in %s, attempt %d/%d", + region, + attempt + 1, + max_retries, + ) + if attempt < max_retries - 1: + sleep_time = retry_delay * ( + 2**attempt + ) # Exponential backoff + logger.info( + "Waiting %.1f seconds before retry...", + sleep_time, + ) + time.sleep(sleep_time) continue - - # Get additional stack metadata - stack_info = self._get_stack_info(cf_client, stack_name) - - export_data = { - "exportName": export_name, - "exportValue": export_value, - "stackName": stack_name, - "stackId": exporting_stack_id, - "region": region, - "description": stack_info.get("description", ""), - "stackStatus": stack_info.get("status", ""), - "creationTime": stack_info.get("creation_time", ""), - "tags": stack_info.get("tags", {}), - "category": self._categorize_export(export_name), - "isUrl": self._is_url_value(export_value), - "consoleUrl": self._get_console_url( - region, - stack_name, - export_name, - export_value, - ), - } - - all_exports.append(export_data) - - except ClientError as e: - logger.error("Error extracting exports from %s: %s", region, e) - continue - except Exception as e: - logger.error("Unexpected error in %s: %s", region, e) - continue + else: + logger.error(error_msg) + + region_errors.append(error_msg) + errors_encountered.append(error_msg) + + if attempt == max_retries - 1: + logger.error( + "Failed to extract exports from %s after %d attempts", + region, + max_retries, + ) + extraction_stats["regions_failed"] += 1 + + except BotoCoreError as e: + error_msg = ( + f"Boto core error in {region} (attempt {attempt + 1}): " + f"{str(e)}" + ) + logger.error(error_msg) + region_errors.append(error_msg) + errors_encountered.append(error_msg) + + if attempt < max_retries - 1: + logger.info(f"Retrying in {retry_delay} seconds...") + time.sleep(retry_delay) + continue + else: + extraction_stats["regions_failed"] += 1 + + except Exception as e: + error_msg = ( + f"Unexpected error in {region} (attempt {attempt + 1}): " + f"{str(e)}" + ) + logger.error(error_msg) + region_errors.append(error_msg) + errors_encountered.append(error_msg) + + if attempt == max_retries - 1: + extraction_stats["regions_failed"] += 1 + + # Log region summary + if region_errors: + logger.warning( + f"Region {region} completed with {len(region_errors)} errors", + ) + else: + logger.info(f"Region {region} completed successfully") # Sort exports by category, then by stack name, then by export name + logger.debug("Sorting exports...") all_exports.sort(key=lambda x: (x["category"], x["stackName"], x["exportName"])) + # Print comprehensive summary + logger.info("=" * 60) + logger.info("EXPORT EXTRACTION SUMMARY") + logger.info("=" * 60) + logger.info(f"Regions scanned: {extraction_stats['regions_scanned']}") + logger.info(f"Regions failed: {extraction_stats['regions_failed']}") + logger.info(f"Total raw exports found: {extraction_stats['total_raw_exports']}") logger.info( - "Extracted %d exports from %d regions", - len(all_exports), - len(regions), + "Exports filtered by prefix: %d", + extraction_stats["filtered_exports"], ) + logger.info( + "Internal exports excluded: %d", + extraction_stats["internal_excluded"], + ) + logger.info("Final exports included: %d", len(all_exports)) + + if errors_encountered: + logger.warning( + "Total errors encountered: %d", + len(errors_encountered), + ) + if logger.getEffectiveLevel() <= logging.DEBUG: + for error in errors_encountered: + logger.debug(" - %s", error) + + # Show breakdown by category + if all_exports: + category_counts: Dict[str, int] = {} + for export in all_exports: + category = export["category"] + category_counts[category] = category_counts.get(category, 0) + 1 + + logger.info("Exports by category:") + for category, count in sorted(category_counts.items()): + logger.info(f" {category}: {count}") + + logger.info("=" * 60) + self.exports_data = all_exports return all_exports @@ -250,6 +600,22 @@ def _categorize_export(self, export_name: str) -> str: # No prefix (no colon), categorize as internal-cdk return "internal-cdk" + def _clean_export_name_for_display(self, export_name: str) -> str: + """ + Clean export name for dashboard display by removing internal prefixes. + + The 'public:' and 'private:' prefixes are used internally for filtering + but should not be shown to users in the dashboard. + """ + # Remove common prefixes used for categorization + prefixes_to_remove = ["public:", "private:"] + + for prefix in prefixes_to_remove: + if export_name.startswith(prefix): + return export_name[len(prefix) :] # noqa: E203 + + return export_name + def _is_url_value(self, value: str) -> bool: """Check if export value appears to be a URL.""" return value.startswith(("http://", "https://")) @@ -262,8 +628,145 @@ def _get_console_url( export_value: str, ) -> str: """Generate AWS Console URL for the export's resource.""" - base_url = "https://console.aws.amazon.com/cloudformation/home" - return f"{base_url}?region={region}#/stacks/stackinfo?stackId={stack_name}" + name_lower = export_name.lower() + + # VPC resources + if "vpc" in name_lower and "vpcid" in name_lower: + vpc_id = export_value + return ( + f"https://console.aws.amazon.com/vpc/home?" + f"region={region}#VpcDetails:VpcId={vpc_id}" + ) + + # S3 Bucket + if "bucket" in name_lower: + bucket_name = ( + export_value.split(":::")[-1] if ":::" in export_value else export_value + ) + return ( + f"https://s3.console.aws.amazon.com/s3/buckets/" + f"{bucket_name}?region={region}" + ) + + # DynamoDB Table + if "dynamodb" in name_lower or "table" in name_lower: + if export_value.startswith("arn:aws:dynamodb"): + table_name = export_value.split("/")[-1] + return ( + f"https://console.aws.amazon.com/dynamodbv2/home?" + f"region={region}#table?name={table_name}" + ) + + # RDS/Aurora + if "aurora" in name_lower or "rds" in name_lower or "cluster" in name_lower: + if "arn:aws:rds" in export_value: + cluster_id = export_value.split(":")[-1] + return ( + f"https://console.aws.amazon.com/rds/home?" + f"region={region}#database:id={cluster_id}" + ) + + # ECS Cluster + if "ecs" in name_lower and "cluster" in name_lower: + if export_value.startswith("arn:aws:ecs"): + cluster_name = export_value.split("/")[-1] + return ( + f"https://console.aws.amazon.com/ecs/v2/clusters/" + f"{cluster_name}?region={region}" + ) + + # EKS Cluster + if "eks" in name_lower and "cluster" in name_lower: + if export_value.startswith("arn:aws:eks"): + cluster_name = export_value.split("/")[-1] + return ( + f"https://console.aws.amazon.com/eks/home?" + f"region={region}#/clusters/{cluster_name}" + ) + + # CloudFront Distribution + if "cloudfront" in name_lower and "distribution" in name_lower: + dist_id = export_value + return ( + f"https://console.aws.amazon.com/cloudfront/v3/home#" + f"/distributions/{dist_id}" + ) + + # SNS Topic + if "sns" in name_lower or "topic" in name_lower: + if export_value.startswith("arn:aws:sns"): + return ( + f"https://console.aws.amazon.com/sns/v3/home?" + f"region={region}#/topic/{export_value}" + ) + + # SQS Queue + if "sqs" in name_lower or "queue" in name_lower: + if export_value.startswith("arn:aws:sqs"): + return ( + f"https://console.aws.amazon.com/sqs/v2/home?" + f"region={region}#/queues/{export_value}" + ) + + # Return empty string for unknown services (will hide the link in template) + return "" + + def _get_stack_info_safe(self, cf_client, stack_name: str, region: str) -> Dict: + """Get additional information about a CloudFormation stack with + enhanced error handling. + """ + try: + logger.debug("Retrieving stack info for %s in %s", stack_name, region) + + response = cf_client.describe_stacks(StackName=stack_name) + stack = response["Stacks"][0] + + # Convert datetime objects to strings + creation_time = stack.get("CreationTime") + if creation_time: + creation_time = creation_time.isoformat() + + # Process tags + tags = {} + for tag in stack.get("Tags", []): + tags[tag["Key"]] = tag["Value"] + + stack_info = { + "description": stack.get("Description", ""), + "status": stack.get("StackStatus", ""), + "creation_time": creation_time, + "tags": tags, + } + + logger.debug( + "Successfully retrieved stack info for %s: %s", + stack_name, + stack_info["status"], + ) + return stack_info + + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + if error_code == "ValidationError": + logger.debug("Stack %s not found or invalid in %s", stack_name, region) + elif error_code in ["AccessDenied", "UnauthorizedOperation"]: + logger.debug("Access denied to stack %s in %s", stack_name, region) + else: + logger.debug( + "API error getting stack info for %s in %s: %s", + stack_name, + region, + error_code, + ) + return {} + except Exception as e: + logger.debug( + "Unexpected error getting stack info for %s in %s: %s", + stack_name, + region, + e, + ) + return {} def _get_stack_info(self, cf_client, stack_name: str) -> Dict: """Get additional information about a CloudFormation stack.""" @@ -322,9 +825,18 @@ def generate_html(self, template_path: Optional[str] = None) -> str: # Fall back to built-in template template = env.from_string(self._get_builtin_template()) + # Prepare exports data with cleaned display names + exports_for_display = [] + for export in self.exports_data: + display_export = export.copy() + display_export["displayName"] = self._clean_export_name_for_display( + export["exportName"], + ) + exports_for_display.append(display_export) + # Prepare template context context = { - "exports": self.exports_data, + "exports": exports_for_display, "generated_at": datetime.now(timezone.utc).isoformat(), "total_exports": len(self.exports_data), "regions": list({export["region"] for export in self.exports_data}), @@ -383,9 +895,9 @@ def upload_to_s3(self, html_content: str, bucket_name: Optional[str] = None) -> # Try to get CloudFront URL if available cloudfront_url = self._get_cloudfront_url(bucket_name, key) - logger.info(f"Uploaded exports dashboard to S3: {s3_url}") # noqa: E501 + logger.info("Uploaded exports dashboard to S3: %s", s3_url) if cloudfront_url: - logger.info(f"CloudFront URL: {cloudfront_url}") # noqa: E501 + logger.info("CloudFront URL: %s", cloudfront_url) return cloudfront_url return s3_url @@ -428,48 +940,68 @@ def _get_cloudfront_url(self, bucket_name: str, key: str) -> Optional[str]: """Try to get CloudFront distribution URL for the S3 bucket.""" try: # Look specifically for WorkshopCloudFrontDomain export first + # Handle both prefixed and non-prefixed export names + domain_export_names = [ + "public:WorkshopCloudFrontDomain", + "WorkshopCloudFrontDomain", + "private:WorkshopCloudFrontDomain", + ] + for export in self.exports_data: - if export["exportName"] == "WorkshopCloudFrontDomain": + if export["exportName"] in domain_export_names: base_url = export["exportValue"] - if base_url.startswith("https://"): - return f"{base_url.rstrip('/')}/{key}" - else: - # Add https if not present + # Ensure it's a domain name, not a distribution ID + if "." in base_url and not base_url.startswith( + ("http://", "https://"), + ): return f"https://{base_url.rstrip('/')}/{key}" + elif base_url.startswith("https://"): + return f"{base_url.rstrip('/')}/{key}" - # Fallback: Look for other CloudFront-related exports + # Fallback: Look for other CloudFront domain-related exports + # But be careful to avoid distribution IDs for export in self.exports_data: - if ( - "cloudfront" in export["exportName"].lower() - or "distribution" in export["exportName"].lower() - ): - base_url = export["exportValue"] - if base_url.startswith("https://"): - return f"{base_url.rstrip('/')}/{key}" - else: - return f"https://{base_url.rstrip('/')}/{key}" + export_name_lower = export["exportName"].lower() + export_value = export["exportValue"] + + if "cloudfront" in export_name_lower and "domain" in export_name_lower: + # Make sure it's a domain name (contains dots) and not + # a distribution ID + if "." in export_value and not export_value.startswith( + ("http://", "https://"), + ): + return f"https://{export_value.rstrip('/')}/{key}" + elif export_value.startswith("https://"): + return f"{export_value.rstrip('/')}/{key}" return None except Exception as e: - logger.debug(f"Could not determine CloudFront URL: {e}") # noqa: E501 + logger.debug("Could not determine CloudFront URL: %s", e) return None def _get_builtin_template(self) -> str: """Return built-in HTML template if external template is not available.""" - return """ # noqa: E501 + return """ CDK Stack Exports Dashboard - - + +