diff --git a/.github/workflows/clean-aws-resources.yml b/.github/workflows/clean-aws-resources.yml index 449aa3dc26..e7463c5c11 100644 --- a/.github/workflows/clean-aws-resources.yml +++ b/.github/workflows/clean-aws-resources.yml @@ -2,7 +2,9 @@ # SPDX-License-Identifier: MIT name: AWS Daily Resources Cleaner - +env: + CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" + CWA_GITHUB_TEST_REPO_BRANCH: "main" on: schedule: - cron: "0 0 * * *" # Run Every Day At Midnight @@ -16,7 +18,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -25,7 +32,7 @@ jobs: aws-region: us-west-2 - name: Clean old ami - working-directory: tool/clean + working-directory: clean run: go run ./clean_ami/clean_ami.go --tags=clean clean-old-file-systems: @@ -35,7 +42,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -44,7 +56,7 @@ jobs: aws-region: us-west-2 - name: Clean old file system - working-directory: tool/clean + working-directory: clean run: go run ./clean_file_system/clean_file_system.go --tags=clean clean-opensource-dedicated-hosts: @@ -54,7 +66,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -63,7 +80,7 @@ jobs: aws-region: us-west-2 - name: Clean old dedicated host - working-directory: tool/clean + working-directory: clean run: go run ./clean_dedicated_host/clean_dedicated_host.go --tags=clean clean-internal-dedicated-hosts: @@ -88,7 +105,12 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -97,7 +119,7 @@ jobs: aws-region: ${{ matrix.region }} - name: Clean old dedicated host - working-directory: tool/clean + working-directory: clean run: go run ./clean_dedicated_host/clean_dedicated_host.go --tags=clean clean-hosts: @@ -128,7 +150,12 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -137,7 +164,7 @@ jobs: aws-region: ${{ matrix.region }} - name: Clean old host - working-directory: tool/clean + working-directory: clean run: go run ./clean_host/clean_host.go ${{ matrix.region }} clean-hosts-china: @@ -147,7 +174,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -156,7 +188,7 @@ jobs: aws-region: "cn-north-1" - name: Clean old hosts - working-directory: tool/clean + working-directory: clean run: go run ./clean_host/clean_host.go cn-north-1 clean-ecs-clusters: @@ -166,7 +198,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -175,7 +212,7 @@ jobs: aws-region: us-west-2 - name: Clean old ecs cluster - working-directory: tool/clean + working-directory: clean run: go run ./clean_ecs/clean_ecs.go --tags=clean clean-eks-clusters: @@ -185,7 +222,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -194,7 +236,7 @@ jobs: aws-region: us-west-2 - name: Clean old eks cluster - working-directory: tool/clean + working-directory: clean run: go run ./clean_eks/clean_eks.go --tags=clean clean-ebs-volumes: runs-on: ubuntu-latest @@ -203,7 +245,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -212,7 +259,7 @@ jobs: aws-region: us-west-2 - name: Clean old unused ebs volumes - working-directory: tool/clean + working-directory: clean run: go run ./clean_ebs/clean_ebs.go --tags=clean clean-asg: @@ -222,7 +269,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -231,7 +283,7 @@ jobs: aws-region: us-west-2 - name: Clean old asg - working-directory: tool/clean + working-directory: clean run: go run ./clean_auto_scaling_groups/clean_auto_scaling_groups.go --tags=clean clean-launch-configs: @@ -241,7 +293,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -250,7 +307,7 @@ jobs: aws-region: us-west-2 - name: Clean old launch configuration - working-directory: tool/clean + working-directory: clean run: go run ./clean_launch_configuration/clean_launch_configuration.go --tags=clean clean-iam-roles: runs-on: ubuntu-latest @@ -259,7 +316,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -268,7 +330,7 @@ jobs: aws-region: us-west-2 - name: Clean old IAM roles - working-directory: tool/clean + working-directory: clean run: go run ./clean_iam_roles/clean_iam_roles.go --tags=clean clean-log-groups: runs-on: ubuntu-latest @@ -277,7 +339,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -286,7 +353,7 @@ jobs: aws-region: us-west-2 - name: Clean old Log Groups - working-directory: tool/clean + working-directory: clean run: go run ./clean_log_group/clean_log_group.go clean-security-groups: runs-on: ubuntu-latest @@ -295,7 +362,12 @@ jobs: contents: read steps: - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -304,7 +376,7 @@ jobs: aws-region: us-west-2 - name: Clean Old Security Groups - working-directory: tool/clean + working-directory: clean run: | set -e go run ./clean_security_group/clean_security_group.go || { echo "Failed to clean security groups"; exit 1; } diff --git a/tool/clean/README.md b/tool/clean/README.md deleted file mode 100644 index 1d0c654b43..0000000000 --- a/tool/clean/README.md +++ /dev/null @@ -1,18 +0,0 @@ -**What does the cleaner do?** - -###Cleaner cleans out old ami (ami older than 60 days) - -The cleaner first searches for ami names (these are the ami created by the pipeline for use int he integration tests) -1. cloudwatch-agent-integration-test* - -Then checks to see if the creation date is greater than 60 days. (The aws sdk v2 gives creation date as a pointer to string. To convert to golang time we use the aws smithy go time. This allows us to compare to 60 days in past time) - -If the ami is older than 60 days old then we delete the ami - -###Cleans dedicated hosts for mac - -The cleaner first searches for dedicated host tag Name:IntegrationTestMacDedicatedHost - -Then checks to see if the creation date is greater than 26 hours and host status is available - -Delete is true \ No newline at end of file diff --git a/tool/clean/clean_ami/clean_ami.go b/tool/clean/clean_ami/clean_ami.go deleted file mode 100644 index 465a41af84..0000000000 --- a/tool/clean/clean_ami/clean_ami.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build clean -// +build clean - -package main - -import ( - "context" - "errors" - "fmt" - "log" - "sort" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - smithyTime "github.com/aws/smithy-go/time" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Image Prefixes are taken from checking the Image Builder Pipelines in us-west-2 -var imagePrefixes = []string{ - "cloudwatch-agent-integration-test-aarch64-al2023", - "cloudwatch-agent-integration-test-al2", - "cloudwatch-agent-integration-test-alma-linux-8", - "cloudwatch-agent-integration-test-alma-linux-9", - "cloudwatch-agent-integration-test-arm64-al2", - "cloudwatch-agent-integration-test-debian-12-arm64", - "cloudwatch-agent-integration-test-nvidia-gpu-al2", - "cloudwatch-agent-integration-test-ol8", - "cloudwatch-agent-integration-test-ol9", - "cloudwatch-agent-integration-test-rocky-linux-8", - "cloudwatch-agent-integration-test-rocky-linux-9", - "cloudwatch-agent-integration-test-sles-15", - "cloudwatch-agent-integration-test-ubuntu-24", - "cloudwatch-agent-integration-test-ubuntu", - "cloudwatch-agent-integration-test-ubuntu-LTS-22", - "cloudwatch-agent-integration-test-win-10", - "cloudwatch-agent-integration-test-win-11", - "cloudwatch-agent-integration-test-win-2016", - "cloudwatch-agent-integration-test-win-2019", - "cloudwatch-agent-integration-test-win-2022", - "cloudwatch-agent-integration-test-x86-al2023", - "cloudwatch-agent-integration-test-mac", - "cloudwatch-agent-integration-test-nvidia-gpu", -} - -func main() { - err := cleanAMIs() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -// takes a list of AMIs and sorts them by creation date (youngest to oldest) -func sortAMIsByCreationDate(amiList []types.Image, errList *[]error) []types.Image { - sort.Slice(amiList, func(i, j int) bool { - if amiList[i].CreationDate != nil && amiList[j].CreationDate != nil { - iCreationDate, iErr := smithyTime.ParseDateTime(*amiList[i].CreationDate) - jCreationDate, jErr := smithyTime.ParseDateTime(*amiList[j].CreationDate) - - if err := errors.Join(iErr, jErr); err != nil && errList != nil { - *errList = append(*errList, err) - return false - } - - return iCreationDate.After(jCreationDate) - } else { - return false - } - }) - - return amiList -} - -// given a slice of AMIs, deregisters them one by one -func deregisterAMIs(ctx context.Context, ec2client *ec2.Client, images []types.Image, errList *[]error) { - for _, image := range images { - if image.Name != nil && image.ImageId != nil && image.CreationDate != nil { - log.Printf("Try to delete ami %v tags %v image id %v image creation date raw %v", *image.Name, image.Tags, *image.ImageId, *image.CreationDate) - deregisterImageInput := &ec2.DeregisterImageInput{ImageId: image.ImageId} - _, err := ec2client.DeregisterImage(ctx, deregisterImageInput) - - if err != nil && errList != nil { - log.Printf("Error while deregistering ami %v", *image.Name) - *errList = append(*errList, err) - } - } - } -} - -// given a map of macos version/architecture to a list of corresponding AMIs, deregister AMIs that are no longer needed -func cleanMacAMIs(ctx context.Context, ec2client *ec2.Client, macosImageAmiMap map[string][]types.Image, expirationDate time.Time, errList *[]error) { - for name, amiList := range macosImageAmiMap { - // don't delete an ami if it's the only one for that version/architecture - if len(amiList) == 1 { - continue - } - - // Sort AMIs by creation date (youngest to oldest) - amiList = sortAMIsByCreationDate(amiList, errList) - - // find the youngest AMI in the list - youngestCreationDate, err := smithyTime.ParseDateTime(aws.ToString(amiList[0].CreationDate)) - - if err != nil && errList != nil { - *errList = append(*errList, err) - continue - } - - if expirationDate.After(youngestCreationDate) { - // If the youngest AMI is over 60 days old, we keep one (the youngest) and can delete the rest - log.Printf("Youngest AMI for %s is over 60 days old. Deleting all but the youngest.", name) - deregisterAMIs(ctx, ec2client, amiList[1:], errList) - } else { - // If the youngest AMI is under 60 days old, keep incrementing until we find AMIs older than 60 days and delete them - for index, ami := range amiList { - creationDate, err := smithyTime.ParseDateTime(aws.ToString(ami.CreationDate)) - if err != nil && errList != nil { - *errList = append(*errList, err) - continue - } - if expirationDate.After(creationDate) { - // once you find the first AMI that's over 60 days old, delete the ones that follow - deregisterAMIs(ctx, ec2client, amiList[index:], errList) - break - } - } - } - } -} - -// given a single non macos image, determine its age and deregister if needed -func cleanNonMacAMIs(ctx context.Context, ec2client *ec2.Client, image types.Image, expirationDate time.Time, errList *[]error) { - creationDate, err := smithyTime.ParseDateTime(aws.ToString(image.CreationDate)) - if err != nil && errList != nil { - *errList = append(*errList, err) - return - } - - if expirationDate.After(creationDate) { - deregisterAMIs(ctx, ec2client, []types.Image{image}, errList) - } -} - -func cleanAMIs() error { - log.Print("Begin to clean EC2 AMI") - - // sets expiration date to 60 days in the past - expirationDate := time.Now().UTC().Add(clean.KeepDurationSixtyDay) - log.Printf("Expiration date set as %v", expirationDate) - - // load default config - ctx := context.Background() - defaultConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err - } - ec2client := ec2.NewFromConfig(defaultConfig) - - // stores a list of AMIs per each macos version/architecture - macosImageAmiMap := make(map[string][]types.Image) - - // Cleanup for each AMI image type - var errList []error - for _, filter := range imagePrefixes { - nameFilter := types.Filter{Name: aws.String("name"), Values: []string{ - fmt.Sprintf("%s*", filter), - }} - - //get instances to delete - describeImagesInput := ec2.DescribeImagesInput{Filters: []types.Filter{nameFilter}} - describeImagesOutput, err := ec2client.DescribeImages(ctx, &describeImagesInput) - if err != nil { - log.Printf("Image filter %s returned an error, skipping :%v", filter, err.Error()) - continue - } - - log.Printf("%s: %d images found", filter, len(describeImagesOutput.Images)) - if len(describeImagesOutput.Images) <= 1 { - log.Printf("1 or less image found for filter %s, skipping", filter) - continue - } - - for _, image := range describeImagesOutput.Images { - if image.Name != nil && filter == "cloudwatch-agent-integration-test-mac" { - // mac image - add it to the map and do nothing else for now - macosImageAmiMap[*image.Name] = append(macosImageAmiMap[*image.Name], image) - } else { - // non mac image - clean it if it's older than 60 days - cleanNonMacAMIs(ctx, ec2client, image, expirationDate, &errList) - } - } - } - - // handle the mac AMIs - cleanMacAMIs(ctx, ec2client, macosImageAmiMap, expirationDate, &errList) - - return nil -} diff --git a/tool/clean/clean_auto_scaling_groups/clean_auto_scaling_groups.go b/tool/clean/clean_auto_scaling_groups/clean_auto_scaling_groups.go deleted file mode 100644 index 4b93f74c07..0000000000 --- a/tool/clean/clean_auto_scaling_groups/clean_auto_scaling_groups.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "log" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/autoscaling" - "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Clean eks clusters if they have been open longer than 7 day -func main() { - err := cleanAutoScalingGroups() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanAutoScalingGroups() error { - log.Print("Begin to clean auto scaling groups Clusters") - ctx := context.Background() - defaultConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err - } - autoScalingClient := autoscaling.NewFromConfig(defaultConfig) - - // filters are and statements not or thus run this 2 times - // wilds cards do not work for asg tags unlike ec2 - ecsFilter := types.Filter{Name: aws.String("tag:BaseClusterName"), Values: []string{ - "cwagent-integ-test-cluster", - }} - eksFilter := types.Filter{Name: aws.String("tag:eks:nodegroup-name"), Values: []string{ - "cwagent-eks-integ-node", - }} - terminateAutoScaling(ctx, autoScalingClient, ecsFilter) - terminateAutoScaling(ctx, autoScalingClient, eksFilter) - return nil -} - -func terminateAutoScaling(ctx context.Context, client *autoscaling.Client, filter types.Filter) { - expirationDateCluster := time.Now().UTC().Add(clean.KeepDurationOneWeek) - describeAutoScalingGroupsInput := autoscaling.DescribeAutoScalingGroupsInput{Filters: []types.Filter{ - filter, - }} - describeAutoScalingGroupsOutput, err := client.DescribeAutoScalingGroups(ctx, &describeAutoScalingGroupsInput) - if err != nil { - log.Fatalf("could not get auto scaling groups") - } - deletePass := 0 - deleteFail := 0 - for _, group := range describeAutoScalingGroupsOutput.AutoScalingGroups { - if expirationDateCluster.After(*group.CreatedTime) { - log.Printf("try to delete auto scaling group %s", *group.AutoScalingGroupName) - deleteAutoScalingGroupInput := autoscaling.DeleteAutoScalingGroupInput{ - AutoScalingGroupName: group.AutoScalingGroupName, - ForceDelete: aws.Bool(true), - } - _, err := client.DeleteAutoScalingGroup(ctx, &deleteAutoScalingGroupInput) - if err != nil { - log.Printf("could not delete auto scaling group %s err %v", *group.AutoScalingGroupName, err) - deleteFail++ - } else { - deletePass++ - } - } - } - log.Printf("was able to delete %d/%d for filter %v", deletePass, deletePass+deleteFail, filter) -} diff --git a/tool/clean/clean_dedicated_host/clean_dedicated_host.go b/tool/clean/clean_dedicated_host/clean_dedicated_host.go deleted file mode 100644 index b4406554e6..0000000000 --- a/tool/clean/clean_dedicated_host/clean_dedicated_host.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build clean -// +build clean - -package main - -import ( - "context" - "log" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Can't release a host if it was being used within the last 24 hr add 2 hr as a buffer -const tagName = "tag:Name" -const tagValue = "IntegrationTestMacDedicatedHost" - -func main() { - err := cleanDedicatedHost() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanDedicatedHost() error { - log.Print("Begin to clean EC2 Dedicated Host") - - expirationDateDedicatedHost := time.Now().UTC().Add(clean.KeepDurationTwentySixHours) - cxt := context.Background() - defaultConfig, err := config.LoadDefaultConfig(cxt) - if err != nil { - return err - } - ec2client := ec2.NewFromConfig(defaultConfig) - - dedicatedHosts, err := getDedicatedHost(cxt, ec2client) - if err != nil { - return err - } - - dedicatedHostIds := make([]string, 0) - for _, dedicatedHost := range dedicatedHosts { - log.Printf("dedicated host id %v experation date %v dedicated host creation date raw %v host state %v", - *dedicatedHost.HostId, expirationDateDedicatedHost, *dedicatedHost.AllocationTime, dedicatedHost.State) - if expirationDateDedicatedHost.After(*dedicatedHost.AllocationTime) && dedicatedHost.State == types.AllocationStateAvailable { - log.Printf("Try to delete dedicated host %s tags %v launch-date %s", *dedicatedHost.HostId, dedicatedHost.Tags, *dedicatedHost.AllocationTime) - dedicatedHostIds = append(dedicatedHostIds, *dedicatedHost.HostId) - } - } - - if len(dedicatedHostIds) == 0 { - log.Printf("No dedicated hosts to release") - return nil - } - - log.Printf("Dedicated hosts to release %v", dedicatedHostIds) - releaseDedicatedHost := ec2.ReleaseHostsInput{HostIds: dedicatedHostIds} - _, err = ec2client.ReleaseHosts(cxt, &releaseDedicatedHost) - return err -} - -func getDedicatedHost(cxt context.Context, ec2client *ec2.Client) ([]types.Host, error) { - // Get list of dedicated host - nameFilter := types.Filter{Name: aws.String(tagName), Values: []string{ - tagValue, - }} - - describeDedicatedHostInput := ec2.DescribeHostsInput{Filter: []types.Filter{nameFilter}} - describeDedicatedHostOutput, err := ec2client.DescribeHosts(cxt, &describeDedicatedHostInput) - if err != nil { - return nil, err - } - return describeDedicatedHostOutput.Hosts, nil -} diff --git a/tool/clean/clean_ebs/clean_ebs.go b/tool/clean/clean_ebs/clean_ebs.go deleted file mode 100644 index 195edb0983..0000000000 --- a/tool/clean/clean_ebs/clean_ebs.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "log" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Clean ebs volumes if they have been open longer than 7 day and unused -func main() { - err := cleanVolumes() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanVolumes() error { - log.Print("Begin to clean EBS Volumes") - ctx := context.Background() - defaultConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err - } - ec2Client := ec2.NewFromConfig(defaultConfig) - - return deleteUnusedVolumes(ctx, ec2Client) - -} - -func deleteUnusedVolumes(ctx context.Context, client *ec2.Client) error { - - input := &ec2.DescribeVolumesInput{ - Filters: []types.Filter{ - { - //if the status is availble, then EBS volume is not currently attached to any ec2 instance (so not being used) - Name: aws.String("status"), - Values: []string{"available"}, - }, - }, - } - - volumes, err := client.DescribeVolumes(ctx, input) - if err != nil { - return err - } - for _, volume := range volumes.Volumes { - if time.Since(*volume.CreateTime) > clean.KeepDurationOneWeek && len(volume.Attachments) == 0 { - log.Printf("Deleting unused volume %s", *volume.VolumeId) - _, err = client.DeleteVolume(ctx, &ec2.DeleteVolumeInput{ - VolumeId: volume.VolumeId, - }) - } - if err != nil { - log.Printf("Error deleting volume %s: %v", *volume.VolumeId, err) - } - } - return nil -} diff --git a/tool/clean/clean_ecs/clean_ecs.go b/tool/clean/clean_ecs/clean_ecs.go deleted file mode 100644 index 2e7b6b74b8..0000000000 --- a/tool/clean/clean_ecs/clean_ecs.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build clean -// +build clean - -package main - -import ( - "context" - "log" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ecs" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Clean ecs clusters if they have been open longer than 7 day -func main() { - err := cleanCluster() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanCluster() error { - log.Print("Begin to clean ECS Clusters") - - cxt := context.Background() - defaultConfig, err := config.LoadDefaultConfig(cxt) - if err != nil { - return err - } - ecsClient := ecs.NewFromConfig(defaultConfig) - - terminateClusters(cxt, ecsClient) - return err -} - -func terminateClusters(ctx context.Context, client *ecs.Client) { - // you can only filter ecs by name or arn - // not regex of tag name like ec2 - // describe cluster input max is 100 - ecsListClusterInput := ecs.ListClustersInput{ - MaxResults: aws.Int32(100), - } - for { - clusterIds := make([]*string, 0) - expirationDateCluster := time.Now().UTC().Add(clean.KeepDurationOneWeek) - listClusterOutput, err := client.ListClusters(ctx, &ecsListClusterInput) - if err != nil || listClusterOutput.ClusterArns == nil || len(listClusterOutput.ClusterArns) == 0 { - break - } - describeClustersInput := ecs.DescribeClustersInput{Clusters: listClusterOutput.ClusterArns} - describeClustersOutput, err := client.DescribeClusters(ctx, &describeClustersInput) - if err != nil || describeClustersOutput.Clusters == nil || len(describeClustersOutput.Clusters) == 0 { - break - } - for _, cluster := range describeClustersOutput.Clusters { - if !strings.HasPrefix(*cluster.ClusterName, "cwagent-integ-test-cluster-") { - continue - } - if cluster.RunningTasksCount == 0 && cluster.PendingTasksCount == 0 { - clusterIds = append(clusterIds, cluster.ClusterArn) - continue - } - describeTaskInput := ecs.DescribeTasksInput{Cluster: cluster.ClusterArn} - describeTasks, err := client.DescribeTasks(ctx, &describeTaskInput) - if err != nil { - continue - } - addCluster := true - for _, task := range describeTasks.Tasks { - if expirationDateCluster.After(*task.StartedAt) { - log.Printf("Task %s launch-date %s", *task.TaskArn, *task.StartedAt) - } else { - addCluster = false - break - } - } - if addCluster { - clusterIds = append(clusterIds, cluster.ClusterArn) - } - } - if len(clusterIds) == 0 { - log.Printf("No clusters to terminate") - return - } - - for _, clusterId := range clusterIds { - log.Printf("cluster to temrinate %s", *clusterId) - listContainerInstanceInput := ecs.ListContainerInstancesInput{Cluster: clusterId} - listContainerInstances, err := client.ListContainerInstances(ctx, &listContainerInstanceInput) - if err != nil { - log.Printf("Error %v getting container instances cluster %s", err, *clusterId) - continue - } - for _, instance := range listContainerInstances.ContainerInstanceArns { - deregisterContainerInstanceInput := ecs.DeregisterContainerInstanceInput{ - ContainerInstance: aws.String(instance), - Cluster: clusterId, - Force: aws.Bool(true), - } - _, err = client.DeregisterContainerInstance(ctx, &deregisterContainerInstanceInput) - if err != nil { - log.Printf("Error %v deregister container instances cluster %s container %v", err, *clusterId, instance) - continue - } - } - serviceInput := ecs.ListServicesInput{Cluster: clusterId} - services, err := client.ListServices(ctx, &serviceInput) - if err != nil { - log.Printf("Error %v getting services cluster %s", err, *clusterId) - continue - } - for _, service := range services.ServiceArns { - deleteServiceInput := ecs.DeleteServiceInput{Cluster: clusterId, Service: aws.String(service)} - _, err := client.DeleteService(ctx, &deleteServiceInput) - if err != nil { - log.Printf("Error %v deleteing service %s cluster %s", err, serviceInput, *clusterId) - continue - } - } - terminateClusterInput := ecs.DeleteClusterInput{Cluster: clusterId} - _, err = client.DeleteCluster(ctx, &terminateClusterInput) - if err != nil { - log.Printf("Error %v terminating cluster %s", err, *clusterId) - } - } - if ecsListClusterInput.NextToken == nil { - break - } - ecsListClusterInput.NextToken = listClusterOutput.NextToken - } -} diff --git a/tool/clean/clean_eks/clean_eks.go b/tool/clean/clean_eks/clean_eks.go deleted file mode 100644 index 087be3622a..0000000000 --- a/tool/clean/clean_eks/clean_eks.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "log" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/eks" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// make this configurable or construct based on some configs? -const betaEksEndpoint = "https://api.beta.us-west-2.wesley.amazonaws.com" - -var ( - ClustersToClean = []string{ - "cwagent-eks-integ-", - "cwagent-operator-helm-integ-", - "cwagent-helm-chart-integ-", - "cwagent-operator-eks-integ-", - "cwagent-monitoring-config-e2e-eks-", - "cwagent-addon-eks-integ-", - } -) - -// Clean eks clusters if they have been open longer than 7 day -func main() { - err := cleanCluster() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanCluster() error { - log.Print("Begin to clean EKS Clusters") - ctx := context.Background() - defaultConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err - } - eksClient := eks.NewFromConfig(defaultConfig) - terminateClusters(ctx, eksClient) - - // delete beta clusters - betaConfig, err := config.LoadDefaultConfig(ctx, config.WithEndpointResolverWithOptions(eksBetaEndpointResolver())) - if err != nil { - return err - } - betaClient := eks.NewFromConfig(betaConfig) - terminateClusters(ctx, betaClient) - - return nil -} - -func eksBetaEndpointResolver() aws.EndpointResolverWithOptionsFunc { - return func(service, region string, options ...interface{}) (aws.Endpoint, error) { - endpoint, err := eks.NewDefaultEndpointResolver().ResolveEndpoint(region, eks.EndpointResolverOptions{}) - if err != nil { - return aws.Endpoint{}, err - } - endpoint.URL = betaEksEndpoint - return endpoint, nil - } -} - -func clusterNameMatchesClustersToClean(clusterName string, clustersToClean []string) bool { - for _, clusterToClean := range clustersToClean { - if strings.HasPrefix(clusterName, clusterToClean) { - return true - } - } - return false -} - -func terminateClusters(ctx context.Context, client *eks.Client) { - listClusterInput := eks.ListClustersInput{} - expirationDateCluster := time.Now().UTC().Add(clean.KeepDurationOneDay) - - clusters, err := client.ListClusters(ctx, &listClusterInput) - if err != nil { - log.Fatalf("could not get cluster list") - } - for _, cluster := range clusters.Clusters { - describeClusterInput := eks.DescribeClusterInput{Name: aws.String(cluster)} - describeClusterOutput, err := client.DescribeCluster(ctx, &describeClusterInput) - if err != nil { - return - } - if !expirationDateCluster.After(*describeClusterOutput.Cluster.CreatedAt) { - log.Printf("Ignoring cluster %s with a launch-date %s since it was created in the last %s", cluster, *describeClusterOutput.Cluster.CreatedAt, clean.KeepDurationOneDay) - continue - } - if !clusterNameMatchesClustersToClean(*describeClusterOutput.Cluster.Name, ClustersToClean) { - log.Printf("Ignoring cluster %s since it doesnt match any of the clean regexes", cluster) - continue - } - log.Printf("Try to delete cluster %s launch-date %s", cluster, *describeClusterOutput.Cluster.CreatedAt) - describeNodegroupInput := eks.ListNodegroupsInput{ClusterName: aws.String(cluster)} - nodeGroupOutput, err := client.ListNodegroups(ctx, &describeNodegroupInput) - if err != nil { - log.Printf("could not query node groups cluster %s err %v", cluster, err) - } - // it takes about 5 minutes to delete node groups - // it will fail to delete cluster if we need to delete node groups - // this will delete the cluster on next run the next day - // I do not want to wait for node groups to be deleted - // as it will increase the runtime of this cleaner - for _, nodegroup := range nodeGroupOutput.Nodegroups { - deleteNodegroupInput := eks.DeleteNodegroupInput{ - ClusterName: aws.String(cluster), - NodegroupName: aws.String(nodegroup), - } - _, err := client.DeleteNodegroup(ctx, &deleteNodegroupInput) - if err != nil { - log.Printf("could not delete node groups %s cluster %s err %v", nodegroup, cluster, err) - } - } - deleteClusterInput := eks.DeleteClusterInput{Name: aws.String(cluster)} - _, err = client.DeleteCluster(ctx, &deleteClusterInput) - if err != nil { - log.Printf("could not delete cluster %s err %v", cluster, err) - } - } -} diff --git a/tool/clean/clean_file_system/clean_file_system.go b/tool/clean/clean_file_system/clean_file_system.go deleted file mode 100644 index f13393ed4b..0000000000 --- a/tool/clean/clean_file_system/clean_file_system.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build clean -// +build clean - -package main - -import ( - "context" - "log" - "time" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/efs" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -func main() { - log.Printf("Begin to clean EFS resources") - expirationDate := time.Now().UTC().Add(clean.KeepDurationOneDay) - cxt := context.Background() - defaultConfig, err := config.LoadDefaultConfig(cxt) - if err != nil { - log.Fatalf("Error getting default config %v", err) - } - efsclient := efs.NewFromConfig(defaultConfig) - - //get efs to delete - var nextToken *string - fileSystemIdSlice := make([]*string, 0) - for { - describeFileSystemsInput := efs.DescribeFileSystemsInput{Marker: nextToken} - describeFileSystemsOutput, err := efsclient.DescribeFileSystems(cxt, &describeFileSystemsInput) - if err != nil { - log.Fatalf("Err %v", err) - } - for _, fileSystem := range describeFileSystemsOutput.FileSystems { - if expirationDate.After(*fileSystem.CreationTime) { - log.Printf("Trying to delete file system %s launch-date %v", *fileSystem.FileSystemId, fileSystem.CreationTime) - if fileSystem.NumberOfMountTargets > 0 { - err = deleteMountTargets(cxt, efsclient, fileSystem.FileSystemId) - } - - if err == nil { - fileSystemIdSlice = append(fileSystemIdSlice, fileSystem.FileSystemId) - } else { - log.Printf("Unable to delete all the mount targets for %s due to %v", *fileSystem.FileSystemId, err) - } - } - } - if describeFileSystemsOutput.NextMarker == nil { - break - } - nextToken = describeFileSystemsOutput.NextMarker - } - for _, fileSystemId := range fileSystemIdSlice { - terminateFileSystemsInput := efs.DeleteFileSystemInput{FileSystemId: fileSystemId} - if _, err = efsclient.DeleteFileSystem(cxt, &terminateFileSystemsInput); err != nil { - log.Printf("Unable to delete file system %s due to %v", *fileSystemId, err) - } else { - log.Printf("Deleted file system %s successfully", *fileSystemId) - } - } -} - -func deleteMountTargets(cxt context.Context, client *efs.Client, fileSystemId *string) error { - var marker *string - for { - dmti := &efs.DescribeMountTargetsInput{Marker: marker, FileSystemId: fileSystemId} - dmto, err := client.DescribeMountTargets(cxt, dmti) - if err != nil { - return err - } - for _, mountTarget := range dmto.MountTargets { - dlmti := &efs.DeleteMountTargetInput{MountTargetId: mountTarget.MountTargetId} - if _, err = client.DeleteMountTarget(cxt, dlmti); err != nil { - return err - } - log.Printf("Deleted mount target %s for %s successfully", *mountTarget.MountTargetId, *fileSystemId) - } - if dmto.Marker == nil { - break - } - marker = dmto.Marker - } - return nil -} diff --git a/tool/clean/clean_host/clean_host.go b/tool/clean/clean_host/clean_host.go deleted file mode 100644 index 31a7d9b5d4..0000000000 --- a/tool/clean/clean_host/clean_host.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT -package main - -import ( - "context" - "log" - "os" - "time" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/aws-sdk-go/aws" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// Clean integration hosts if they have been open longer than 1 day -func main() { - err := cleanHost() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} - -func cleanHost() error { - log.Print("Begin to clean EC2 Host") - - cxt := context.Background() - defaultConfig, err := config.LoadDefaultConfig(cxt, config.WithRegion(os.Args[1])) - if err != nil { - return err - } - ec2client := ec2.NewFromConfig(defaultConfig) - - terminateInstances(cxt, ec2client) - return err -} - -func terminateInstances(cxt context.Context, ec2client *ec2.Client) { - maxResults := int32(1000) - nameFilter := types.Filter{Name: aws.String("tag:Name"), Values: []string{ - "buildLinuxPackage", - "buildPKG", - "buildMSI", - "MSIUpgrade_*", - "Ec2IntegrationTest", - "IntegrationTestBase", - "CWADockerImageBuilderX86", - "CWADockerImageBuilderARM64", - "cwagent-integ-test-ec2*", - "cwagent-integ-test-ec2-windows-*", - "cwagent-performance-*", - "cwagent-stress-*", - "LocalStackIntegrationTestInstance", - "NvidiaDataCollector-*", - }} - - instanceInput := ec2.DescribeInstancesInput{ - Filters: []types.Filter{ - nameFilter, - {Name: aws.String("instance-state-name"), - Values: []string{"running"}}}, - MaxResults: aws.Int32(maxResults)} - for { - instanceIds := make([]string, 0) - expirationDateInstance := time.Now().UTC().Add(clean.KeepDurationOneDay) - describeInstanceOutput, _ := ec2client.DescribeInstances(cxt, &instanceInput) - for _, reservation := range describeInstanceOutput.Reservations { - for _, instance := range reservation.Instances { - log.Printf("instance id %v expiration date %v host creation date raw %v host state %v", - *instance.InstanceId, expirationDateInstance, *instance.LaunchTime, instance.State) - if expirationDateInstance.After(*instance.LaunchTime) { - log.Printf("Try to delete instance %s tags %v launch-date %s", *instance.InstanceId, instance.Tags, *instance.LaunchTime) - instanceIds = append(instanceIds, *instance.InstanceId) - } - } - } - if len(instanceIds) == 0 { - log.Printf("No instances to terminate") - return - } - - log.Printf("instances to terminate %v", instanceIds) - terminateInstance := ec2.TerminateInstancesInput{InstanceIds: instanceIds} - _, err := ec2client.TerminateInstances(cxt, &terminateInstance) - if err != nil { - log.Printf("Error %v terminating instances %v", err, instanceIds) - } - if describeInstanceOutput.NextToken == nil { - break - } - instanceInput.NextToken = describeInstanceOutput.NextToken - // prevent throttle https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html - time.Sleep(time.Minute) - } -} diff --git a/tool/clean/clean_iam_roles/clean_iam_roles.go b/tool/clean/clean_iam_roles/clean_iam_roles.go deleted file mode 100644 index c8ced60401..0000000000 --- a/tool/clean/clean_iam_roles/clean_iam_roles.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "errors" - "fmt" - "log" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/iam/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -const retentionPeriod = clean.KeepDurationOneWeek - -var ( - roleNamePrefixes = []string{ - "cwa-integ-assume-role", - "cwagent-eks-Worker-Role", - "cwagent-integ-test-task-role", - "cwagent-integ-test-task-execution-role", - "cwagent-operator-eks-Worker-Role", - "cwagent-operator-helm-integ-Worker-Role", - } -) - -type iamClient interface { - ListRoles(ctx context.Context, input *iam.ListRolesInput, optFns ...func(*iam.Options)) (*iam.ListRolesOutput, error) - GetRole(ctx context.Context, input *iam.GetRoleInput, optFns ...func(*iam.Options)) (*iam.GetRoleOutput, error) - DeleteRole(ctx context.Context, input *iam.DeleteRoleInput, optFns ...func(*iam.Options)) (*iam.DeleteRoleOutput, error) - ListAttachedRolePolicies(ctx context.Context, input *iam.ListAttachedRolePoliciesInput, optFns ...func(*iam.Options)) (*iam.ListAttachedRolePoliciesOutput, error) - DetachRolePolicy(ctx context.Context, input *iam.DetachRolePolicyInput, optFns ...func(*iam.Options)) (*iam.DetachRolePolicyOutput, error) - ListInstanceProfilesForRole(ctx context.Context, input *iam.ListInstanceProfilesForRoleInput, optFns ...func(*iam.Options)) (*iam.ListInstanceProfilesForRoleOutput, error) - RemoveRoleFromInstanceProfile(ctx context.Context, input *iam.RemoveRoleFromInstanceProfileInput, optFns ...func(*iam.Options)) (*iam.RemoveRoleFromInstanceProfileOutput, error) - DeleteInstanceProfile(ctx context.Context, input *iam.DeleteInstanceProfileInput, optFns ...func(*iam.Options)) (*iam.DeleteInstanceProfileOutput, error) -} - -func main() { - log.Print("Begin to clean IAM Roles") - ctx := context.Background() - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - log.Fatalf("unable to load AWS config: %v", err) - } - client := iam.NewFromConfig(cfg) - if err = deleteRoles(ctx, client, getExpirationDate()); err != nil { - log.Printf("errors cleaning: %v\n", err) - } -} - -func getExpirationDate() time.Time { - return time.Now().UTC().Add(retentionPeriod) -} - -func deleteRoles(ctx context.Context, client iamClient, expirationDate time.Time) error { - var errs error - var marker *string - for { - output, err := client.ListRoles(ctx, &iam.ListRolesInput{Marker: marker}) - if err != nil { - return err - } - for _, role := range output.Roles { - if hasPrefix(*role.RoleName) && expirationDate.After(*role.CreateDate) { - var getRoleOutput *iam.GetRoleOutput - getRoleOutput, err = client.GetRole(ctx, &iam.GetRoleInput{RoleName: role.RoleName}) - if err != nil { - return err - } - role = *getRoleOutput.Role - if role.RoleLastUsed == nil || role.RoleLastUsed.LastUsedDate == nil || expirationDate.After(*role.RoleLastUsed.LastUsedDate) { - errs = errors.Join(errs, deleteRole(ctx, client, role)) - } - } - } - if output.Marker == nil { - break - } - marker = output.Marker - } - return errs -} - -func deleteRole(ctx context.Context, client iamClient, role types.Role) error { - lastUsed := "never" - if role.RoleLastUsed != nil && role.RoleLastUsed.LastUsedDate != nil { - lastUsed = fmt.Sprintf("%d days ago", int(time.Since(*role.RoleLastUsed.LastUsedDate).Hours()/24)) - } - log.Printf("Trying to delete role (%q) last used %s", *role.RoleName, lastUsed) - if err := detachPolicies(ctx, client, role); err != nil { - return err - } - if err := deleteProfiles(ctx, client, role); err != nil { - return err - } - if _, err := client.DeleteRole(ctx, &iam.DeleteRoleInput{RoleName: role.RoleName}); err != nil { - log.Printf("Failed to delete role (%q): %v", *role.RoleName, err) - return err - } - log.Printf("Deleted role (%q) successfully", *role.RoleName) - return nil -} - -func detachPolicies(ctx context.Context, client iamClient, role types.Role) error { - var marker *string - for { - output, err := client.ListAttachedRolePolicies(ctx, &iam.ListAttachedRolePoliciesInput{RoleName: role.RoleName, Marker: marker}) - if err != nil { - return err - } - for _, policy := range output.AttachedPolicies { - log.Printf("Trying to detach policy (%q) from role (%q)", *policy.PolicyName, *role.RoleName) - if _, err = client.DetachRolePolicy(ctx, &iam.DetachRolePolicyInput{PolicyArn: policy.PolicyArn, RoleName: role.RoleName}); err != nil { - log.Printf("Failed to detach policy (%q): %v", *policy.PolicyName, err) - return err - } - log.Printf("Detached policy (%q) from role (%q) successfully", *policy.PolicyName, *role.RoleName) - } - if output.Marker == nil { - break - } - marker = output.Marker - } - return nil -} - -func deleteProfiles(ctx context.Context, client iamClient, role types.Role) error { - var marker *string - for { - output, err := client.ListInstanceProfilesForRole(ctx, &iam.ListInstanceProfilesForRoleInput{RoleName: role.RoleName, Marker: marker}) - if err != nil { - return err - } - for _, profile := range output.InstanceProfiles { - if err = deleteProfile(ctx, client, role, profile); err != nil { - return err - } - } - if output.Marker == nil { - break - } - marker = output.Marker - } - return nil -} - -func deleteProfile(ctx context.Context, client iamClient, role types.Role, profile types.InstanceProfile) error { - log.Printf("Trying to remove role (%q) from instance profile (%q)", *role.RoleName, *profile.InstanceProfileName) - _, err := client.RemoveRoleFromInstanceProfile(ctx, &iam.RemoveRoleFromInstanceProfileInput{ - InstanceProfileName: profile.InstanceProfileName, - RoleName: role.RoleName, - }) - if err != nil { - return err - } - log.Printf("Removed role (%q) from instance profile (%q) successfully", *role.RoleName, *profile.InstanceProfileName) - // profile's only role is about to be deleted, so delete it too - if len(profile.Roles) == 1 { - log.Printf("Trying to delete instance profile (%q) attached to role (%q)", *profile.InstanceProfileName, *role.RoleName) - if _, err = client.DeleteInstanceProfile(ctx, &iam.DeleteInstanceProfileInput{InstanceProfileName: profile.InstanceProfileName}); err != nil { - log.Printf("Failed to delete instance profile (%q): %v", *profile.InstanceProfileName, err) - return err - } - log.Printf("Deleted instance profile (%q) successfully", *profile.InstanceProfileName) - } - return nil -} - -func hasPrefix(roleName string) bool { - for _, prefix := range roleNamePrefixes { - if strings.HasPrefix(roleName, prefix) { - return true - } - } - return false -} diff --git a/tool/clean/clean_iam_roles/clean_iam_roles_test.go b/tool/clean/clean_iam_roles/clean_iam_roles_test.go deleted file mode 100644 index 8fb0f6219b..0000000000 --- a/tool/clean/clean_iam_roles/clean_iam_roles_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -const ( - expiredTestRoleName = "cwa-integ-assume-role-expired" - activeTestRoleName = "cwagent-integ-test-task-role-active" -) - -type mockIamClient struct { - mock.Mock -} - -var _ iamClient = (*mockIamClient)(nil) - -func (m *mockIamClient) ListRoles(ctx context.Context, input *iam.ListRolesInput, optFns ...func(*iam.Options)) (*iam.ListRolesOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.ListRolesOutput), args.Error(1) -} - -func (m *mockIamClient) GetRole(ctx context.Context, input *iam.GetRoleInput, optFns ...func(*iam.Options)) (*iam.GetRoleOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.GetRoleOutput), args.Error(1) -} - -func (m *mockIamClient) DeleteRole(ctx context.Context, input *iam.DeleteRoleInput, optFns ...func(*iam.Options)) (*iam.DeleteRoleOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.DeleteRoleOutput), args.Error(1) -} - -func (m *mockIamClient) ListAttachedRolePolicies(ctx context.Context, input *iam.ListAttachedRolePoliciesInput, optFns ...func(*iam.Options)) (*iam.ListAttachedRolePoliciesOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.ListAttachedRolePoliciesOutput), args.Error(1) -} - -func (m *mockIamClient) DetachRolePolicy(ctx context.Context, input *iam.DetachRolePolicyInput, optFns ...func(*iam.Options)) (*iam.DetachRolePolicyOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.DetachRolePolicyOutput), args.Error(1) -} - -func (m *mockIamClient) ListInstanceProfilesForRole(ctx context.Context, input *iam.ListInstanceProfilesForRoleInput, optFns ...func(*iam.Options)) (*iam.ListInstanceProfilesForRoleOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.ListInstanceProfilesForRoleOutput), args.Error(1) -} - -func (m *mockIamClient) RemoveRoleFromInstanceProfile(ctx context.Context, input *iam.RemoveRoleFromInstanceProfileInput, optFns ...func(*iam.Options)) (*iam.RemoveRoleFromInstanceProfileOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.RemoveRoleFromInstanceProfileOutput), args.Error(1) -} - -func (m *mockIamClient) DeleteInstanceProfile(ctx context.Context, input *iam.DeleteInstanceProfileInput, optFns ...func(*iam.Options)) (*iam.DeleteInstanceProfileOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*iam.DeleteInstanceProfileOutput), args.Error(1) -} - -func TestDeleteRoles(t *testing.T) { - expirationDate := getExpirationDate() - - ctx := context.Background() - ignoredRole := types.Role{ - CreateDate: aws.Time(expirationDate.Add(-2 * time.Hour)), - RoleName: aws.String("does-not-match-any-prefix"), - RoleLastUsed: &types.RoleLastUsed{ - LastUsedDate: aws.Time(expirationDate.Add(-1 * time.Hour)), - }, - } - expiredRole := types.Role{ - CreateDate: aws.Time(expirationDate.Add(-2 * time.Hour)), - RoleName: aws.String(expiredTestRoleName), - RoleLastUsed: &types.RoleLastUsed{ - LastUsedDate: aws.Time(expirationDate.Add(-1 * time.Hour)), - }, - } - activeRole := types.Role{ - CreateDate: aws.Time(expirationDate.Add(-2 * time.Hour)), - RoleName: aws.String(activeTestRoleName), - RoleLastUsed: &types.RoleLastUsed{ - LastUsedDate: aws.Time(expirationDate.Add(time.Hour)), - }, - } - testRoles := []types.Role{ignoredRole, expiredRole, activeRole} - testPolicies := []types.AttachedPolicy{ - { - PolicyArn: aws.String("policy-arn"), - PolicyName: aws.String("policy-name"), - }, - } - testProfile := types.InstanceProfile{ - InstanceProfileName: aws.String("instance-profile-name"), - Roles: []types.Role{expiredRole}, - } - - client := &mockIamClient{} - client.On("ListRoles", ctx, &iam.ListRolesInput{}, mock.Anything).Return(&iam.ListRolesOutput{Roles: testRoles}, nil) - client.On("GetRole", ctx, &iam.GetRoleInput{RoleName: aws.String(expiredTestRoleName)}, mock.Anything).Return(&iam.GetRoleOutput{Role: &expiredRole}, nil) - client.On("GetRole", ctx, &iam.GetRoleInput{RoleName: aws.String(activeTestRoleName)}, mock.Anything).Return(&iam.GetRoleOutput{Role: &activeRole}, nil) - client.On("DeleteRole", ctx, mock.Anything, mock.Anything).Return(&iam.DeleteRoleOutput{}, nil) - client.On("ListAttachedRolePolicies", ctx, mock.Anything, mock.Anything).Return(&iam.ListAttachedRolePoliciesOutput{ - AttachedPolicies: testPolicies, - }, nil) - client.On("DetachRolePolicy", ctx, mock.Anything, mock.Anything).Return(&iam.DetachRolePolicyOutput{}, nil) - client.On("ListInstanceProfilesForRole", ctx, &iam.ListInstanceProfilesForRoleInput{ - RoleName: aws.String(expiredTestRoleName), - }, mock.Anything).Return(&iam.ListInstanceProfilesForRoleOutput{InstanceProfiles: []types.InstanceProfile{testProfile}}, nil) - client.On("RemoveRoleFromInstanceProfile", ctx, &iam.RemoveRoleFromInstanceProfileInput{ - RoleName: aws.String(expiredTestRoleName), - InstanceProfileName: testProfile.InstanceProfileName, - }, mock.Anything).Return(&iam.RemoveRoleFromInstanceProfileOutput{}, nil) - client.On("DeleteInstanceProfile", ctx, &iam.DeleteInstanceProfileInput{ - InstanceProfileName: testProfile.InstanceProfileName, - }, mock.Anything).Return(&iam.DeleteInstanceProfileOutput{}, nil) - assert.NoError(t, deleteRoles(ctx, client, expirationDate)) - assert.Len(t, client.Calls, 9) - for _, call := range client.Calls { - switch call.Method { - case "DeleteRole": - input := call.Arguments.Get(1).(*iam.DeleteRoleInput) - assert.Equal(t, expiredTestRoleName, *input.RoleName) - case "ListAttachedRolePolicies": - input := call.Arguments.Get(1).(*iam.ListAttachedRolePoliciesInput) - assert.Equal(t, expiredTestRoleName, *input.RoleName) - case "DetachRolePolicy": - input := call.Arguments.Get(1).(*iam.DetachRolePolicyInput) - assert.Equal(t, expiredTestRoleName, *input.RoleName) - assert.Equal(t, "policy-arn", *input.PolicyArn) - } - } -} diff --git a/tool/clean/clean_launch_configuration/clean_launch_configuration.go b/tool/clean/clean_launch_configuration/clean_launch_configuration.go deleted file mode 100644 index 4c52cc4872..0000000000 --- a/tool/clean/clean_launch_configuration/clean_launch_configuration.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT -package main - -import ( - "context" - "errors" - "log" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/autoscaling" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -func main() { - err := cleanLaunchConfiguration() - if err != nil { - log.Fatalf("errors cleaning %v", err) - } -} -func cleanLaunchConfiguration() error { - expirationDate := time.Now().UTC().Add(clean.KeepDurationSixtyDay) - ctx := context.Background() - defaultConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err - } - autoScalingClient := autoscaling.NewFromConfig(defaultConfig) - describeLaunchConfigurationsInput := autoscaling.DescribeLaunchConfigurationsInput{} - launchConfigOut, err := autoScalingClient.DescribeLaunchConfigurations(ctx, &describeLaunchConfigurationsInput) - if err != nil { - return err - } - if len(launchConfigOut.LaunchConfigurations) == 0 { - return errors.New("no launch configuration found") - } - log.Printf("Found %d launch configurations", len(launchConfigOut.LaunchConfigurations)) - for _, launchConfig := range launchConfigOut.LaunchConfigurations { - log.Printf("Found %s with creation date: %v", *launchConfig.LaunchConfigurationName, *launchConfig.CreatedTime) - // if not cwagent-integ-test ignore it - if !strings.Contains(*launchConfig.LaunchConfigurationName, "integ") { - continue - } - if expirationDate.After(*launchConfig.CreatedTime) { - log.Printf("Try to delete %s", *launchConfig.LaunchConfigurationName) - _, err := autoScalingClient.DeleteLaunchConfiguration(ctx, &autoscaling.DeleteLaunchConfigurationInput{ - LaunchConfigurationName: launchConfig.LaunchConfigurationName, - }) - if err != nil { - return err - } - log.Printf("Succesfully deleted %s", *launchConfig.LaunchConfigurationName) - } - } - return nil -} diff --git a/tool/clean/clean_log_group/clean_log_group.go b/tool/clean/clean_log_group/clean_log_group.go deleted file mode 100644 index e7674d9623..0000000000 --- a/tool/clean/clean_log_group/clean_log_group.go +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "flag" - "fmt" - "log" - "strings" - "sync" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -type cloudwatchlogsClient interface { - DeleteLogGroup(ctx context.Context, params *cloudwatchlogs.DeleteLogGroupInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DeleteLogGroupOutput, error) - DescribeLogGroups(ctx context.Context, params *cloudwatchlogs.DescribeLogGroupsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) - DescribeLogStreams(ctx context.Context, params *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) -} - -const ( - LogGroupProcessChanSize = 500 -) - -// Config holds the application configuration -type Config struct { - creationThreshold time.Duration - inactiveThreshold time.Duration - numWorkers int - deleteBatchCap int - exceptionList []string - dryRun bool -} - -// Global configuration -var ( - cfg Config -) - -func init() { - // Set default configuration - cfg = Config{ - creationThreshold: 3 * clean.KeepDurationOneDay, - inactiveThreshold: 1 * clean.KeepDurationOneDay, - numWorkers: 15, - exceptionList: []string{"lambda"}, - dryRun: true, - } - -} - -func main() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - // Parse command line flags - flag.BoolVar(&cfg.dryRun, "dry-run", false, "Enable dry-run mode (no actual deletion)") - flag.Parse() - // Load AWS configuration - awsCfg, err := loadAWSConfig(ctx) - if err != nil { - log.Fatalf("Error loading AWS config: %v", err) - } - - // Create CloudWatch Logs client - client := cloudwatchlogs.NewFromConfig(awsCfg) - - // Compute cutoff times - cutoffTimes := calculateCutoffTimes() - - log.Printf("🔍 Searching for CloudWatch Log Groups older than %d days AND inactive for %d days in %s region\n", - cfg.creationThreshold, cfg.inactiveThreshold, awsCfg.Region) - - // Delete old log groups - deletedGroups := deleteOldLogGroups(ctx, client, cutoffTimes) - log.Printf("Total log groups deleted: %d", len(deletedGroups)) -} - -type cutoffTimes struct { - creation int64 - inactive int64 -} - -func calculateCutoffTimes() cutoffTimes { - return cutoffTimes{ - creation: time.Now().Add(cfg.creationThreshold).UnixMilli(), - inactive: time.Now().Add(cfg.inactiveThreshold).UnixMilli(), - } -} - -func loadAWSConfig(ctx context.Context) (aws.Config, error) { - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return aws.Config{}, fmt.Errorf("loading AWS config: %w", err) - } - cfg.RetryMode = aws.RetryModeAdaptive - return cfg, nil -} - -func deleteOldLogGroups(ctx context.Context, client cloudwatchlogsClient, times cutoffTimes) []string { - var ( - wg sync.WaitGroup - deletedLogGroup []string - foundLogGroupChan = make(chan *types.LogGroup, LogGroupProcessChanSize) - deletedLogGroupNameChan = make(chan string, LogGroupProcessChanSize) - handlerWg sync.WaitGroup - ) - - // Start worker pool - log.Printf("👷 Creating %d workers\n", cfg.numWorkers) - for i := 0; i < cfg.numWorkers; i++ { - wg.Add(1) - w := worker{ - id: i, - wg: &wg, - incomingLogGroupChan: foundLogGroupChan, - deletedLogGroupChan: deletedLogGroupNameChan, - times: times, - } - go w.processLogGroup(ctx, client) - } - - // Start handler with its own WaitGroup - handlerWg.Add(1) - go func() { - handleDeletedLogGroups(&deletedLogGroup, deletedLogGroupNameChan) - handlerWg.Done() - }() - - // Process log groups in batches - if err := fetchAndProcessLogGroups(ctx, client, foundLogGroupChan); err != nil { - log.Printf("Error processing log groups: %v", err) - } - - close(foundLogGroupChan) - wg.Wait() - close(deletedLogGroupNameChan) - handlerWg.Wait() - - return deletedLogGroup -} - -func handleDeletedLogGroups(deletedLogGroups *[]string, deletedLogGroupNameChan chan string) { - for logGroupName := range deletedLogGroupNameChan { - *deletedLogGroups = append(*deletedLogGroups, logGroupName) - log.Printf("🔍 Processed %d log groups so far\n", len(*deletedLogGroups)) - } -} - -type worker struct { - id int - wg *sync.WaitGroup - incomingLogGroupChan <-chan *types.LogGroup - deletedLogGroupChan chan<- string - times cutoffTimes -} - -func (w *worker) processLogGroup(ctx context.Context, client cloudwatchlogsClient) { - defer w.wg.Done() - - for logGroup := range w.incomingLogGroupChan { - if err := w.handleLogGroup(ctx, client, logGroup); err != nil { - log.Printf("Worker %d: Error processing log group: %v", w.id, err) - } - } -} - -func (w *worker) handleLogGroup(ctx context.Context, client cloudwatchlogsClient, logGroup *types.LogGroup) error { - if logGroup.CreationTime == nil { - return fmt.Errorf("log group has no creation time: %v", logGroup) - } - - logGroupName := *logGroup.LogGroupName - creationTime := *logGroup.CreationTime - - if creationTime >= w.times.creation { - return nil - } - - lastLogTime := getLastLogEventTime(ctx, client, logGroupName) - if lastLogTime == 0 { - return nil - } - - if lastLogTime < w.times.inactive { - log.Printf("🚨 Worker: %d| Old & Inactive Log Group: %s (Created: %v, Last Event: %v)\n", - w.id, logGroupName, time.Unix(creationTime, 0), time.Unix(lastLogTime, 0)) - - w.deletedLogGroupChan <- logGroupName - - if cfg.dryRun { - log.Printf("🛑 Dry-Run: Would delete log group: %s", logGroupName) - return nil - } - - return deleteLogGroup(ctx, client, logGroupName) - } - - return nil -} - -func deleteLogGroup(ctx context.Context, client cloudwatchlogsClient, logGroupName string) error { - _, err := client.DeleteLogGroup(ctx, &cloudwatchlogs.DeleteLogGroupInput{ - LogGroupName: aws.String(logGroupName), - }) - if err != nil { - return fmt.Errorf("deleting log group %s: %w", logGroupName, err) - } - log.Printf("✅ Deleted log group: %s", logGroupName) - return nil -} - -func fetchAndProcessLogGroups(ctx context.Context, client cloudwatchlogsClient, - logGroupChan chan<- *types.LogGroup) error { - - var nextToken *string - describeCount := 0 - - for { - output, err := client.DescribeLogGroups(ctx, &cloudwatchlogs.DescribeLogGroupsInput{ - NextToken: nextToken, - }) - if err != nil { - return fmt.Errorf("describing log groups: %w", err) - } - - log.Printf("🔍 Described %d times | Found %d log groups\n", describeCount, len(output.LogGroups)) - - for _, logGroup := range output.LogGroups { - if isLogGroupException(*logGroup.LogGroupName) { - log.Printf("⏭️ Skipping Log Group: %s (in exception list)\n", *logGroup.LogGroupName) - continue - } - logGroupChan <- &logGroup - } - - if output.NextToken == nil { - break - } - - nextToken = output.NextToken - describeCount++ - } - - return nil -} - -func getLastLogEventTime(ctx context.Context, client cloudwatchlogsClient, logGroupName string) int64 { - var latestTimestamp int64 - latestTimestamp = 0 - output, err := client.DescribeLogStreams(ctx, &cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String(logGroupName), - OrderBy: types.OrderByLastEventTime, - Descending: aws.Bool(true), - }) - if err != nil { - log.Printf("⚠️ Warning: Failed to retrieve log streams for %s: %v\n", logGroupName, err) - return 0 - } - if len(output.LogStreams) == 0 { - return 0 - } - stream := output.LogStreams[0] - - if stream.LastEventTimestamp != nil && *stream.LastEventTimestamp > latestTimestamp { - latestTimestamp = *stream.LastEventTimestamp - } - - return latestTimestamp -} - -func isLogGroupException(logGroupName string) bool { - for _, exception := range cfg.exceptionList { - if strings.Contains(logGroupName, exception) { - return true - } - } - return false -} diff --git a/tool/clean/clean_log_group/clean_log_group_test.go b/tool/clean/clean_log_group/clean_log_group_test.go deleted file mode 100644 index 580ba0071a..0000000000 --- a/tool/clean/clean_log_group/clean_log_group_test.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -// main_test.go -package main - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -// MockCloudWatchLogsClient is a stub for cloudwatchlogs.Client -type MockCloudWatchLogsClient struct { - mock.Mock -} - -var _ cloudwatchlogsClient = (*MockCloudWatchLogsClient)(nil) - -func (m *MockCloudWatchLogsClient) DescribeLogGroups(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*cloudwatchlogs.DescribeLogGroupsOutput), args.Error(1) -} - -func (m *MockCloudWatchLogsClient) DeleteLogGroup(ctx context.Context, input *cloudwatchlogs.DeleteLogGroupInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DeleteLogGroupOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*cloudwatchlogs.DeleteLogGroupOutput), args.Error(1) -} - -func (m *MockCloudWatchLogsClient) DescribeLogStreams(ctx context.Context, input *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) { - args := m.Called(ctx, input, optFns) - return args.Get(0).(*cloudwatchlogs.DescribeLogStreamsOutput), args.Error(1) -} - -// Test getLastLogEventTime simulating multiple pages of log streams. -func TestGetLastLogEventTime(t *testing.T) { - mockClient := new(MockCloudWatchLogsClient) - - // Create two pages of responses - firstPage := &cloudwatchlogs.DescribeLogStreamsOutput{ - LogStreams: []types.LogStream{ - {LastEventTimestamp: aws.Int64(1000)}, - {LastEventTimestamp: aws.Int64(1500)}, - }, - NextToken: aws.String("token1"), - } - secondPage := &cloudwatchlogs.DescribeLogStreamsOutput{ - LogStreams: []types.LogStream{ - {LastEventTimestamp: aws.Int64(2000)}, - {LastEventTimestamp: aws.Int64(1800)}, - }, - NextToken: nil, - } - - // Set up expectations for the two API calls - mockClient.On("DescribeLogStreams", - mock.Anything, - &cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String("dummy-log-group"), - OrderBy: types.OrderByLastEventTime, - Descending: aws.Bool(true), - NextToken: nil, - }, - mock.Anything).Return(firstPage, nil).Once() - - mockClient.On("DescribeLogStreams", - mock.Anything, - &cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String("dummy-log-group"), - OrderBy: types.OrderByLastEventTime, - Descending: aws.Bool(true), - NextToken: aws.String("token1"), - }, - mock.Anything).Return(secondPage, nil).Once() - - lastEventTime := getLastLogEventTime(context.Background(), mockClient, "dummy-log-group") - - assert.Equal(t, int64(1000), lastEventTime) -} -func testHandleLogGroup(cfg Config, logGroupName string, logCreationDate, logStreamCreationDate int) ([]string, error) { - cfg.dryRun = true // Prevent actual deletion. - - // Calculate cutoffs relative to now. - now := time.Now() - times := cutoffTimes{ - creation: now.Add(cfg.creationThreshold).UnixMilli(), - inactive: now.Add(cfg.inactiveThreshold).UnixMilli(), - } - // Create a dummy log group. - creationTime := now.AddDate(0, 0, -logCreationDate).UnixMilli() - logGroup := &types.LogGroup{ - LogGroupName: aws.String(logGroupName), - CreationTime: aws.Int64(creationTime), - } - - mockClient := new(MockCloudWatchLogsClient) - - // Set up expectation for DescribeLogStreams - mockClient.On("DescribeLogStreams", - mock.Anything, - &cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String(logGroupName), - OrderBy: types.OrderByLastEventTime, - Descending: aws.Bool(true), - NextToken: nil, - }, - mock.Anything).Return(&cloudwatchlogs.DescribeLogStreamsOutput{ - LogStreams: []types.LogStream{ - {LastEventTimestamp: aws.Int64(now.AddDate(0, 0, -logStreamCreationDate).UnixMilli())}, - }, - NextToken: nil, - }, nil).Once() - - var ( - deletedLogGroup []string - wg sync.WaitGroup - foundLogGroupChan = make(chan *types.LogGroup, 500) - deletedLogGroupNameChan = make(chan string, 500) - ) - - w := worker{ - id: 1, - wg: &wg, - incomingLogGroupChan: foundLogGroupChan, - deletedLogGroupChan: deletedLogGroupNameChan, - times: times, - } - go handleDeletedLogGroups(&deletedLogGroup, deletedLogGroupNameChan) - // Call handleLogGroup in dry-run mode (so no deletion call is made). - err := w.handleLogGroup(context.Background(), mockClient, logGroup) - time.Sleep(1 * time.Second) // give time for deleted group to be handled - return deletedLogGroup, err - -} - -// Test handleLogGroup to simulate deletion when a log group is old and inactive. -func TestHandleLogGroup(t *testing.T) { - cfg := Config{ - creationThreshold: 3 * clean.KeepDurationOneDay, - inactiveThreshold: 2 * clean.KeepDurationOneDay, - numWorkers: 0, - deleteBatchCap: 0, - exceptionList: []string{"EXCEPTION"}, - dryRun: true, - } - testCases := []struct { - name string - logGroupName string - logCreationDate int - logStreamCreationDate int - expected []string - }{ - { - "Expired log group", - "expired-test-log-group", - 7, - 7, - []string{"expired-test-log-group"}, - }, - { - "Fresh log group", - "fresh-test-log-group", - 1, - 7, - []string{}, - }, - { - "Old but still used log group", - "old-test-log-group", - 7, - 1, - []string{}, - }, - } - for _, tc := range testCases { - t.Run(tc.logGroupName, func(t *testing.T) { - deletedLogGroup, err := testHandleLogGroup(cfg, tc.logGroupName, tc.logCreationDate, tc.logStreamCreationDate) - assert.NoError(t, err) - assert.Len(t, deletedLogGroup, len(tc.expected)) - assert.ElementsMatch(t, deletedLogGroup, tc.expected) - }) - } - -} -func testDeleteLogGroup(cfg Config, logGroupName string, logCreationDate, logStreamCreationDate int) []string { - cfg.dryRun = true // Prevent actual deletion. - now := time.Now() - - mockClient := new(MockCloudWatchLogsClient) - - // Set up expectation for DescribeLogGroups - mockClient.On("DescribeLogGroups", - mock.Anything, - &cloudwatchlogs.DescribeLogGroupsInput{ - NextToken: nil, - }, - mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{ - LogGroups: []types.LogGroup{ - {LogGroupName: aws.String(logGroupName), - CreationTime: aws.Int64(now.AddDate(0, 0, -logCreationDate).UnixMilli())}, - }, - NextToken: nil, - }, nil).Once() - mockClient.On("DescribeLogStreams", - mock.Anything, - &cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: aws.String(logGroupName), - OrderBy: types.OrderByLastEventTime, - Descending: aws.Bool(true), - NextToken: nil, - }, - mock.Anything).Return(&cloudwatchlogs.DescribeLogStreamsOutput{ - LogStreams: []types.LogStream{ - {LastEventTimestamp: aws.Int64(now.AddDate(0, 0, -logStreamCreationDate).UnixMilli())}, - }, - NextToken: nil, - }, nil).Once() - - // Call handleLogGroup in dry-run mode (so no deletion call is made). - return deleteOldLogGroups(context.Background(), mockClient, calculateCutoffTimes()) - -} -func TestDeleteLogGroups(t *testing.T) { - cfg := Config{ - creationThreshold: 3, - inactiveThreshold: 2, - numWorkers: 0, - deleteBatchCap: 0, - exceptionList: []string{"except"}, - dryRun: true, - } - testCases := []struct { - name string - logGroupName string - logCreationDate int - logStreamCreationDate int - expected []string - }{ - { - "Expired log group", - "expired-test-log-group", - 7, - 7, - []string{"expired-test-log-group"}, - }, - { - "Fresh log group", - "fresh-test-log-group", - 1, - 7, - []string{}, - }, - { - "Old but still used log group", - "old-test-log-group", - 7, - 1, - []string{}, - }, - { - "Exception log group", - "exceptional-test-log-group", - 7, - 1, - []string{}, - }, - } - for _, tc := range testCases { - t.Run(tc.logGroupName, func(t *testing.T) { - deletedLogGroup := testDeleteLogGroup(cfg, tc.logGroupName, tc.logCreationDate, tc.logStreamCreationDate) - assert.Len(t, deletedLogGroup, len(tc.expected)) - assert.ElementsMatch(t, deletedLogGroup, tc.expected) - }) - } - -} diff --git a/tool/clean/clean_security_group/clean_security_group.go b/tool/clean/clean_security_group/clean_security_group.go deleted file mode 100644 index 1e550b099d..0000000000 --- a/tool/clean/clean_security_group/clean_security_group.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "flag" - "fmt" - "log" - "strings" - "sync" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - - "github.com/aws/amazon-cloudwatch-agent/tool/clean" -) - -type ec2Client interface { - DescribeSecurityGroups(ctx context.Context, params *ec2.DescribeSecurityGroupsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error) - DescribeNetworkInterfaces(ctx context.Context, params *ec2.DescribeNetworkInterfacesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) - DeleteSecurityGroup(ctx context.Context, params *ec2.DeleteSecurityGroupInput, optFns ...func(*ec2.Options)) (*ec2.DeleteSecurityGroupOutput, error) - RevokeSecurityGroupIngress(ctx context.Context, params *ec2.RevokeSecurityGroupIngressInput, optFns ...func(*ec2.Options)) (*ec2.RevokeSecurityGroupIngressOutput, error) - RevokeSecurityGroupEgress(ctx context.Context, params *ec2.RevokeSecurityGroupEgressInput, optFns ...func(*ec2.Options)) (*ec2.RevokeSecurityGroupEgressOutput, error) -} - -const ( - SecurityGroupProcessChanSize = 500 -) - -// Config holds the application configuration -type Config struct { - ageThreshold time.Duration - numWorkers int - exceptionList []string - dryRun bool - skipVpcSGs bool - skipWithRules bool -} - -// Global configuration -var ( - cfg Config -) - -func init() { - // Set default configuration - cfg = Config{ - ageThreshold: 1 * clean.KeepDurationOneDay, - numWorkers: 30, - exceptionList: []string{"default"}, - dryRun: true, - skipVpcSGs: false, - skipWithRules: false, - } -} - -func main() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - // Parse command line flags - flag.BoolVar(&cfg.dryRun, "dry-run", true, "Enable dry-run mode (no actual deletion)") - flag.DurationVar(&cfg.ageThreshold, "age", 1*clean.KeepDurationOneDay, "Age threshold for security groups (e.g. 24h)") - flag.BoolVar(&cfg.skipVpcSGs, "skip-vpc", false, "Skip security groups associated with VPCs") - flag.BoolVar(&cfg.skipWithRules, "skip-with-rules", false, "Skip security groups that have ingress or egress rules") - flag.Parse() - - // Load AWS configuration - awsCfg, err := loadAWSConfig(ctx) - if err != nil { - log.Fatalf("Error loading AWS config: %v", err) - } - - // Create EC2 client - client := ec2.NewFromConfig(awsCfg) - - log.Printf("🔍 Searching for unused Security Groups older than %v in %s region\n", - cfg.ageThreshold, awsCfg.Region) - - // Delete old security groups - deletedGroups, err := deleteUnusedSecurityGroups(ctx, client) - if err != nil { - log.Printf("Error deleting security groups: %v", err) - } - log.Printf("Total security groups deleted: %d", len(deletedGroups)) -} - -func loadAWSConfig(ctx context.Context) (aws.Config, error) { - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return aws.Config{}, fmt.Errorf("loading AWS config: %w", err) - } - cfg.RetryMode = aws.RetryModeAdaptive - return cfg, nil -} - -func deleteUnusedSecurityGroups(ctx context.Context, client ec2Client) ([]string, error) { - var ( - wg sync.WaitGroup - deletedSecurityGroups []string - foundSecurityGroupChan = make(chan types.SecurityGroup, SecurityGroupProcessChanSize) - deletedSecurityGroupChan = make(chan string, SecurityGroupProcessChanSize) - handlerWg sync.WaitGroup - ) - - // Start worker pool - log.Printf("👷 Creating %d workers\n", cfg.numWorkers) - for i := 0; i < cfg.numWorkers; i++ { - wg.Add(1) - w := worker{ - id: i, - wg: &wg, - incomingSecurityGroupChan: foundSecurityGroupChan, - deletedSecurityGroupChan: deletedSecurityGroupChan, - } - go w.processSecurityGroup(ctx, client) - } - - // Start handler with its own WaitGroup - handlerWg.Add(1) - go func() { - handleDeletedSecurityGroups(&deletedSecurityGroups, deletedSecurityGroupChan) - handlerWg.Done() - }() - - // Process security groups in batches - if err := fetchAndProcessSecurityGroups(ctx, client, foundSecurityGroupChan); err != nil { - log.Printf("Error processing security groups: %v", err) - return nil, err - } - - close(foundSecurityGroupChan) - wg.Wait() - close(deletedSecurityGroupChan) - handlerWg.Wait() - - return deletedSecurityGroups, nil -} - -func handleDeletedSecurityGroups(deletedSecurityGroups *[]string, deletedSecurityGroupChan chan string) { - for securityGroupId := range deletedSecurityGroupChan { - *deletedSecurityGroups = append(*deletedSecurityGroups, securityGroupId) - log.Printf("🔍 Processed %d security groups so far\n", len(*deletedSecurityGroups)) - } -} - -type worker struct { - id int - wg *sync.WaitGroup - incomingSecurityGroupChan <-chan types.SecurityGroup - deletedSecurityGroupChan chan<- string -} - -func (w *worker) processSecurityGroup(ctx context.Context, client ec2Client) { - defer w.wg.Done() - - for { - select { - case securityGroup, ok := <-w.incomingSecurityGroupChan: - if !ok { - return - } - if err := w.handleSecurityGroup(ctx, client, securityGroup); err != nil { - log.Printf("Worker %d: Error processing security group: %v", w.id, err) - } - case <-ctx.Done(): - log.Printf("Worker %d: Stopping due to context cancellation", w.id) - return - } - } -} - -func (w *worker) handleSecurityGroup(ctx context.Context, client ec2Client, securityGroup types.SecurityGroup) error { - sgID := *securityGroup.GroupId - sgName := *securityGroup.GroupName - - // Skip default security groups - if isDefaultSecurityGroup(securityGroup) { - log.Printf("⏭️ Worker %d: Skipping default security group: %s (%s)", w.id, sgID, sgName) - return nil - } - - // Skip security groups in exception list - if isSecurityGroupException(securityGroup) { - log.Printf("⏭️ Worker %d: Skipping security group in exception list: %s (%s)", w.id, sgID, sgName) - return nil - } - - // Check if security group is in use - isInUse, err := isSecurityGroupInUse(ctx, client, sgID) - if err != nil { - return fmt.Errorf("checking if security group is in use: %w", err) - } - - if isInUse { - log.Printf("⏭️ Worker %d: Security group is in use: %s (%s)", w.id, sgID, sgName) - return nil - } - - // Check if security group has rules and we're configured to skip those - if cfg.skipWithRules && hasRules(securityGroup) { - log.Printf("⏭️ Worker %d: Skipping security group with rules: %s (%s)", w.id, sgID, sgName) - return nil - } - - log.Printf("🚨 Worker %d: Found unused security group: %s (%s)", w.id, sgID, sgName) - - // Clean up any rules before deletion - if hasRules(securityGroup) { - if err := cleanSecurityGroupRules(ctx, client, securityGroup); err != nil { - return fmt.Errorf("cleaning security group rules: %w", err) - } - } - - w.deletedSecurityGroupChan <- sgID - - if cfg.dryRun { - log.Printf("🛑 Dry-Run: Would delete security group: %s (%s)", sgID, sgName) - return nil - } - - return deleteSecurityGroup(ctx, client, sgID) -} - -func deleteSecurityGroup(ctx context.Context, client ec2Client, securityGroupID string) error { - _, err := client.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{ - GroupId: aws.String(securityGroupID), - }) - if err != nil { - return fmt.Errorf("deleting security group %s: %w", securityGroupID, err) - } - log.Printf("✅ Deleted security group: %s", securityGroupID) - return nil -} - -func cleanSecurityGroupRules(ctx context.Context, client ec2Client, securityGroup types.SecurityGroup) error { - sgID := *securityGroup.GroupId - - // Get fresh security group data in one call - describeOutput, err := client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - GroupIds: []string{sgID}, - }) - if err != nil { - return fmt.Errorf("describing security group %s: %w", sgID, err) - } - - if len(describeOutput.SecurityGroups) == 0 { - return fmt.Errorf("security group %s not found", sgID) - } - - sg := describeOutput.SecurityGroups[0] - - // Handle both ingress and egress rules concurrently - var wg sync.WaitGroup - var ingressErr, egressErr error - - if len(sg.IpPermissions) > 0 { - if cfg.dryRun { - log.Printf("🛑 Dry-Run: Would revoke %d ingress rules from security group: %s", - len(sg.IpPermissions), sgID) - } else { - wg.Add(1) - go func() { - defer wg.Done() - _, err := client.RevokeSecurityGroupIngress(ctx, &ec2.RevokeSecurityGroupIngressInput{ - GroupId: aws.String(sgID), - IpPermissions: sg.IpPermissions, - }) - if err != nil { - ingressErr = fmt.Errorf("revoking ingress rules: %w", err) - } else { - log.Printf("✅ Revoked ingress rules from security group: %s", sgID) - } - }() - } - } - - if len(sg.IpPermissionsEgress) > 0 { - if cfg.dryRun { - log.Printf("🛑 Dry-Run: Would revoke %d egress rules from security group: %s", - len(sg.IpPermissionsEgress), sgID) - } else { - wg.Add(1) - go func() { - defer wg.Done() - _, err := client.RevokeSecurityGroupEgress(ctx, &ec2.RevokeSecurityGroupEgressInput{ - GroupId: aws.String(sgID), - IpPermissions: sg.IpPermissionsEgress, - }) - if err != nil { - egressErr = fmt.Errorf("revoking egress rules: %w", err) - } else { - log.Printf("✅ Revoked egress rules from security group: %s", sgID) - } - }() - } - } - - wg.Wait() - - if ingressErr != nil { - return ingressErr - } - if egressErr != nil { - return egressErr - } - - return nil -} - -func fetchAndProcessSecurityGroups(ctx context.Context, client ec2Client, - securityGroupChan chan<- types.SecurityGroup) error { - - maxResults := int32(100) // AWS maximum allowed - var nextToken *string - describeCount := 0 - - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - output, err := client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - MaxResults: aws.Int32(maxResults), - NextToken: nextToken, - }) - if err != nil { - return fmt.Errorf("describing security groups: %w", err) - } - - log.Printf("🔍 Described %d times | Found %d security groups\n", describeCount, len(output.SecurityGroups)) - - // Process in batches with context awareness - for _, securityGroup := range output.SecurityGroups { - select { - case securityGroupChan <- securityGroup: - case <-ctx.Done(): - return ctx.Err() - } - } - - if output.NextToken == nil { - break - } - - nextToken = output.NextToken - describeCount++ - } - } - - return nil -} - -func isSecurityGroupInUse(ctx context.Context, client ec2Client, securityGroupID string) (bool, error) { - // Use a channel to handle concurrent checks - resultChan := make(chan bool, 2) - errChan := make(chan error, 2) - - // Check network interfaces concurrently - go func() { - output, err := client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ - Filters: []types.Filter{ - { - Name: aws.String("group-id"), - Values: []string{securityGroupID}, - }, - }, - }) - if err != nil { - errChan <- fmt.Errorf("describing network interfaces: %w", err) - return - } - resultChan <- len(output.NetworkInterfaces) > 0 - }() - - // Check security group references concurrently - go func() { - output, err := client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{}) - if err != nil { - errChan <- fmt.Errorf("describing security groups: %w", err) - return - } - - for _, sg := range output.SecurityGroups { - if *sg.GroupId == securityGroupID { - continue - } - - // Check both ingress and egress rules - if isReferencedInRules(sg.IpPermissions, securityGroupID) || - isReferencedInRules(sg.IpPermissionsEgress, securityGroupID) { - resultChan <- true - return - } - } - resultChan <- false - }() - - // Wait for both checks - for i := 0; i < 2; i++ { - select { - case err := <-errChan: - return false, err - case isUsed := <-resultChan: - if isUsed { - return true, nil - } - case <-ctx.Done(): - return false, ctx.Err() - } - } - - return false, nil -} - -func isReferencedInRules(permissions []types.IpPermission, securityGroupID string) bool { - for _, permission := range permissions { - for _, userIdGroupPair := range permission.UserIdGroupPairs { - if userIdGroupPair.GroupId != nil && *userIdGroupPair.GroupId == securityGroupID { - return true - } - } - } - return false -} - -func isDefaultSecurityGroup(securityGroup types.SecurityGroup) bool { - return *securityGroup.GroupName == "default" -} - -func isSecurityGroupException(securityGroup types.SecurityGroup) bool { - sgName := *securityGroup.GroupName - for _, exception := range cfg.exceptionList { - if strings.Contains(sgName, exception) { - return true - } - } - return false -} - -func hasRules(securityGroup types.SecurityGroup) bool { - return len(securityGroup.IpPermissions) > 0 || len(securityGroup.IpPermissionsEgress) > 0 -} diff --git a/tool/clean/clean_security_group/clean_security_group_test.go b/tool/clean/clean_security_group/clean_security_group_test.go deleted file mode 100644 index db66194b09..0000000000 --- a/tool/clean/clean_security_group/clean_security_group_test.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "sync" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/stretchr/testify/assert" -) - -// Mock EC2 client for testing -type mockEC2Client struct { - describeSecurityGroupsOutput *ec2.DescribeSecurityGroupsOutput - describeNetworkInterfacesOutput *ec2.DescribeNetworkInterfacesOutput - deleteSecurityGroupCalled bool - deleteSecurityGroupInput *ec2.DeleteSecurityGroupInput - revokeIngressCalled bool - revokeEgressCalled bool -} - -func (m *mockEC2Client) DescribeSecurityGroups(ctx context.Context, params *ec2.DescribeSecurityGroupsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error) { - if params != nil && len(params.GroupIds) > 0 { - // When called with specific GroupIds, return only those groups - matchingGroups := []types.SecurityGroup{} - for _, sg := range m.describeSecurityGroupsOutput.SecurityGroups { - for _, requestedID := range params.GroupIds { - if *sg.GroupId == requestedID { - matchingGroups = append(matchingGroups, sg) - break - } - } - } - return &ec2.DescribeSecurityGroupsOutput{ - SecurityGroups: matchingGroups, - }, nil - } - return m.describeSecurityGroupsOutput, nil -} - -func (m *mockEC2Client) DescribeNetworkInterfaces(ctx context.Context, params *ec2.DescribeNetworkInterfacesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeNetworkInterfacesOutput, error) { - return m.describeNetworkInterfacesOutput, nil -} - -func (m *mockEC2Client) DeleteSecurityGroup(ctx context.Context, params *ec2.DeleteSecurityGroupInput, optFns ...func(*ec2.Options)) (*ec2.DeleteSecurityGroupOutput, error) { - m.deleteSecurityGroupCalled = true - m.deleteSecurityGroupInput = params - return &ec2.DeleteSecurityGroupOutput{}, nil -} - -func (m *mockEC2Client) RevokeSecurityGroupIngress(ctx context.Context, params *ec2.RevokeSecurityGroupIngressInput, optFns ...func(*ec2.Options)) (*ec2.RevokeSecurityGroupIngressOutput, error) { - m.revokeIngressCalled = true - return &ec2.RevokeSecurityGroupIngressOutput{}, nil -} - -func (m *mockEC2Client) RevokeSecurityGroupEgress(ctx context.Context, params *ec2.RevokeSecurityGroupEgressInput, optFns ...func(*ec2.Options)) (*ec2.RevokeSecurityGroupEgressOutput, error) { - m.revokeEgressCalled = true - return &ec2.RevokeSecurityGroupEgressOutput{}, nil -} - -func TestIsDefaultSecurityGroup(t *testing.T) { - tests := []struct { - name string - securityGroup types.SecurityGroup - expected bool - }{ - { - name: "Default security group", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-12345"), - GroupName: aws.String("default"), - }, - expected: true, - }, - { - name: "Non-default security group", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-67890"), - GroupName: aws.String("my-security-group"), - }, - expected: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := isDefaultSecurityGroup(tt.securityGroup) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestIsSecurityGroupException(t *testing.T) { - // Set up test exception list - cfg.exceptionList = []string{"default", "special"} - - tests := []struct { - name string - securityGroup types.SecurityGroup - expected bool - }{ - { - name: "Security group in exception list", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-12345"), - GroupName: aws.String("special-group"), - }, - expected: true, - }, - { - name: "Security group not in exception list", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-67890"), - GroupName: aws.String("my-security-group"), - }, - expected: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := isSecurityGroupException(tt.securityGroup) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestHasRules(t *testing.T) { - tests := []struct { - name string - securityGroup types.SecurityGroup - expected bool - }{ - { - name: "Security group with ingress rules", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-12345"), - GroupName: aws.String("test-group"), - IpPermissions: []types.IpPermission{ - { - IpProtocol: aws.String("tcp"), - FromPort: aws.Int32(22), - ToPort: aws.Int32(22), - }, - }, - }, - expected: true, - }, - { - name: "Security group with egress rules", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-67890"), - GroupName: aws.String("test-group"), - IpPermissionsEgress: []types.IpPermission{ - { - IpProtocol: aws.String("-1"), - }, - }, - }, - expected: true, - }, - { - name: "Security group with no rules", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-abcde"), - GroupName: aws.String("test-group"), - }, - expected: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := hasRules(tt.securityGroup) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestHandleSecurityGroup(t *testing.T) { - // Save original dry run setting and restore after test - originalDryRun := cfg.dryRun - defer func() { cfg.dryRun = originalDryRun }() - - tests := []struct { - name string - securityGroup types.SecurityGroup - dryRun bool - hasNetworkInterfaces bool - expectDelete bool - expectRevokeRules bool - }{ - { - name: "Default security group - should not delete", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-12345"), - GroupName: aws.String("default"), - }, - dryRun: false, - hasNetworkInterfaces: false, - expectDelete: false, - expectRevokeRules: false, - }, - { - name: "Security group with network interfaces - should not delete", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-67890"), - GroupName: aws.String("test-group"), - }, - dryRun: false, - hasNetworkInterfaces: true, - expectDelete: false, - expectRevokeRules: false, - }, - { - name: "Unused security group with rules - should delete and revoke rules", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-abcde"), - GroupName: aws.String("test-group"), - IpPermissions: []types.IpPermission{ - { - IpProtocol: aws.String("tcp"), - FromPort: aws.Int32(22), - ToPort: aws.Int32(22), - }, - }, - IpPermissionsEgress: []types.IpPermission{ - { - IpProtocol: aws.String("-1"), - }, - }, - }, - dryRun: false, - hasNetworkInterfaces: false, - expectDelete: true, - expectRevokeRules: true, - }, - { - name: "Dry run - should not actually delete", - securityGroup: types.SecurityGroup{ - GroupId: aws.String("sg-abcde"), - GroupName: aws.String("test-group"), - }, - dryRun: true, - hasNetworkInterfaces: false, - expectDelete: false, - expectRevokeRules: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg.dryRun = tt.dryRun - - // Create mock client - mockClient := &mockEC2Client{ - describeNetworkInterfacesOutput: &ec2.DescribeNetworkInterfacesOutput{ - NetworkInterfaces: make([]types.NetworkInterface, 0), - }, - describeSecurityGroupsOutput: &ec2.DescribeSecurityGroupsOutput{ - SecurityGroups: []types.SecurityGroup{tt.securityGroup}, - }, - } - - if tt.hasNetworkInterfaces { - mockClient.describeNetworkInterfacesOutput.NetworkInterfaces = append( - mockClient.describeNetworkInterfacesOutput.NetworkInterfaces, - types.NetworkInterface{ - NetworkInterfaceId: aws.String("eni-12345"), - }, - ) - } - - // Create worker - w := worker{ - id: 1, - wg: &sync.WaitGroup{}, - deletedSecurityGroupChan: make(chan string, 1), - } - - // Call the function - err := w.handleSecurityGroup(context.Background(), mockClient, tt.securityGroup) - - // Check results - assert.NoError(t, err) - assert.Equal(t, tt.expectDelete && !tt.dryRun, mockClient.deleteSecurityGroupCalled) - assert.Equal(t, tt.expectRevokeRules && !tt.dryRun, mockClient.revokeIngressCalled || mockClient.revokeEgressCalled) - - // Clean up channel - close(w.deletedSecurityGroupChan) - }) - } -} diff --git a/tool/clean/clean_util.go b/tool/clean/clean_util.go deleted file mode 100644 index 903907e342..0000000000 --- a/tool/clean/clean_util.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package clean - -import "time" - -const ( - KeepDurationOneWeek = KeepDurationOneDay * 7 - KeepDurationFourDays = KeepDurationOneDay * 4 - KeepDurationOneDay = -1 * time.Hour * 24 - KeepDurationSixtyDay = KeepDurationOneDay * time.Duration(60) - KeepDurationTwentySixHours = KeepDurationOneDay + time.Hour*2 -) diff --git a/tool/clean/go.mod b/tool/clean/go.mod deleted file mode 100644 index e2209f0071..0000000000 --- a/tool/clean/go.mod +++ /dev/null @@ -1,39 +0,0 @@ -module github.com/aws/amazon-cloudwatch-agent/tool/clean - -go 1.22 - -toolchain go1.23.6 - -require ( - github.com/aws/aws-sdk-go v1.48.14 - github.com/aws/aws-sdk-go-v2 v1.36.2 - github.com/aws/aws-sdk-go-v2/config v1.25.12 - github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3 - github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.140.0 - github.com/aws/aws-sdk-go-v2/service/ecs v1.35.3 - github.com/aws/aws-sdk-go-v2/service/efs v1.26.3 - github.com/aws/aws-sdk-go-v2/service/eks v1.35.3 - github.com/aws/aws-sdk-go-v2/service/iam v1.28.3 - github.com/aws/smithy-go v1.22.2 - github.com/stretchr/testify v1.8.4 -) - -require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.16.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/tool/clean/go.sum b/tool/clean/go.sum deleted file mode 100644 index ec10d06adb..0000000000 --- a/tool/clean/go.sum +++ /dev/null @@ -1,70 +0,0 @@ -github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s= -github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.36.2 h1:Ub6I4lq/71+tPb/atswvToaLGVMxKZvjYDVOWEExOcU= -github.com/aws/aws-sdk-go-v2 v1.36.2/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0= -github.com/aws/aws-sdk-go-v2/config v1.25.12/go.mod h1:lOvvqtZP9p29GIjOTuA/76HiVk0c/s8qRcFRq2+E2uc= -github.com/aws/aws-sdk-go-v2/credentials v1.16.10 h1:VmRkuoKaGl2ZDNGkkRQgw80Hxj1Bb9a+bsT5shqlCwo= -github.com/aws/aws-sdk-go-v2/credentials v1.16.10/go.mod h1:WEn22lpd50buTs/TDqywytW5xQ2zPOMbYipIlqI6xXg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 h1:knLyPMw3r3JsU8MFHWctE4/e2qWbPaxDYLlohPvnY8c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33/go.mod h1:EBp2HQ3f+XCB+5J+IoEbGhoV7CpJbnrsd4asNXmTL0A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 h1:K0+Ne08zqti8J9jwENxZ5NoUyBnaFDTu3apwQJWrwwA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33/go.mod h1:K97stwwzaWzmqxO8yLGHhClbVW1tC6VT1pDLk1pGrq4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3 h1:16TRfDZhx5aX90VsvG0yJ5XNlDNHMVDj2DBpVMwDxzc= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3/go.mod h1:dJD5FZKnDClUVIcCwfu676Y72h4GytmMoZHzf1nTW8Q= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14 h1:Xc90sglbEnAC1X4d4ui422Ppw0HWjyNoqGAE1Dq+Rcg= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14/go.mod h1:IbPFVuHnR+Klb3rrZHai890N1dnMCJZ0GeRfG0fj+ys= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.140.0 h1:joMAX3jOjpbgIYzXgyMLAYly0kzbTJ7DrfAB3PNwobA= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.140.0/go.mod h1:d1hAqgLDOPaSO1Piy/0bBmj6oAplFwv6p0cquHntNHM= -github.com/aws/aws-sdk-go-v2/service/ecs v1.35.3 h1:5P4kia3F4RC5/nBQDBlH0WijReF9YIZ9JwvwoVBRZxY= -github.com/aws/aws-sdk-go-v2/service/ecs v1.35.3/go.mod h1:MvDz+yXfa2sSEfHB57rdf83deKJIeKEopqHFhVmaRlk= -github.com/aws/aws-sdk-go-v2/service/efs v1.26.3 h1:8P0NaanIGybEl84KOoSf3mK7iR2wVCaM4VBlXTcwHCY= -github.com/aws/aws-sdk-go-v2/service/efs v1.26.3/go.mod h1:eWEZMOIwD29RZjTYqN66v0pRoicZbJCyk8RqZ0PgqfA= -github.com/aws/aws-sdk-go-v2/service/eks v1.35.3 h1:l7ucCZKpubBcFA8cOtHsnE98CqFq4KdTf+7+/aCAh+g= -github.com/aws/aws-sdk-go-v2/service/eks v1.35.3/go.mod h1:HZ5xGhvvZEs7iBPbwhLWZWvAxWezzOFn/HrepDivCgQ= -github.com/aws/aws-sdk-go-v2/service/iam v1.28.3 h1:vk0prYDicp3+/Mh+ihU7VPvVMjnfmzseRO/xjOqthPs= -github.com/aws/aws-sdk-go-v2/service/iam v1.28.3/go.mod h1:KJbw+8r7gZfjF+OewOVhyEQKiJXJ/OM1F1r3aAvKS9M= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.3 h1:wKspi1zc2ZVcgZEu3k2Mt4zGKQSoZTftsoUTLsYPcVo= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.3/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 h1:CxAHBS0BWSUqI7qzXHc2ZpTeHaM9JNnWJ9BN6Kmo2CY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 h1:KfREzajmHCSYjCaMRtdLr9boUMA7KPpoPApitPlbNeo= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.3/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=