Merge branch 'dev' into branch-eni-config

Author: JoseVillalta

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+# 1.89.1
+* Changes included from discarded v1.89.0 release
+
+# 1.89.0
+* Enhancement - Bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /ecs-agent/daemonimages/csidriver [#4361](https://github.com/aws/amazon-ecs-agent/pull/4361)
+* Enhancement - Log unresolved dependencies at info level [#4419](https://github.com/aws/amazon-ecs-agent/pull/4419)
+* Enhancement - Make the path to resolv.conf a configuration parameter instead of a hardcoded string [#4421](https://github.com/aws/amazon-ecs-agent/pull/4421)
+* Bugfix - Implement reference counting of volume mounts in amazon-ecs-volume-plugin [#4425](https://github.com/aws/amazon-ecs-agent/pull/4425) 
+* Bugfix - Set firelens mem_buf_limit by default [#4405](https://github.com/aws/amazon-ecs-agent/pull/4405)
+
 # 1.88.0
 * Feature - Fault Injection Service Integration [#4414](https://github.com/aws/amazon-ecs-agent/pull/4414)
 * Bugfix - Retry GPU devices check during env vars load if instance supports GPU [#4387](https://github.com/aws/amazon-ecs-agent/pull/4387)

Makefile

@@ -30,8 +30,6 @@ else
 	GO_VERSION=$(shell cat ./GO_VERSION)
 endif
 
-VERSION=$(shell cat VERSION)
-
 export GO111MODULE=auto
 
 all: docker
@@ -401,6 +399,7 @@ get-deps-init:
 	GO111MODULE=on go install honnef.co/go/tools/cmd/[email protected]
 
 amazon-linux-sources.tgz:
+	./scripts/update-version.sh
 	cp packaging/amazon-linux-ami-integrated/ecs-agent.spec ecs-agent.spec
 	cp packaging/amazon-linux-ami-integrated/ecs.conf ecs.conf
 	cp packaging/amazon-linux-ami-integrated/ecs.service ecs.service
@@ -411,14 +410,15 @@ amazon-linux-sources.tgz:
 
 .amazon-linux-rpm-integrated-done: amazon-linux-sources.tgz
 	test -e SOURCES || ln -s . SOURCES
-	rpmbuild --define "%version ${VERSION}" --define "%_topdir $(PWD)" -bb ecs-agent.spec
+	rpmbuild --define "%_topdir $(PWD)" -bb ecs-agent.spec
 	find RPMS/ -type f -exec cp {} . \;
 	touch .amazon-linux-rpm-integrated-done
 
 amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done
 
 # Make target for Amazon Linux Codebuild jobs
 .amazon-linux-rpm-codebuild-done: get-cni-sources
+	./scripts/update-version.sh
 	cp packaging/amazon-linux-ami-integrated/ecs-agent.spec ecs-agent.spec
 	cp packaging/amazon-linux-ami-integrated/ecs.conf ecs.conf
 	cp packaging/amazon-linux-ami-integrated/ecs.service ecs.service
@@ -427,27 +427,31 @@ amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done
 	cp packaging/amazon-linux-ami-integrated/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket
 	tar -czf ./sources.tgz ecs-init scripts misc agent amazon-ecs-cni-plugins amazon-vpc-cni-plugins agent-container Makefile VERSION GO_VERSION
 	test -e SOURCES || ln -s . SOURCES
-	rpmbuild --define "%version ${VERSION}" --define "%_topdir $(PWD)" -bb ecs-agent.spec
+	rpmbuild --define "%_topdir $(PWD)" -bb ecs-agent.spec
 	find RPMS/ -type f -exec cp {} . \;
 	touch .amazon-linux-rpm-codebuild-done
 
 amazon-linux-rpm-codebuild: .amazon-linux-rpm-codebuild-done
 
 .generic-rpm-integrated-done: get-cni-sources
+	./scripts/update-version.sh
 	cp packaging/generic-rpm-integrated/amazon-ecs-init.spec amazon-ecs-init.spec
 	cp packaging/generic-rpm-integrated/ecs.service ecs.service
 	cp packaging/generic-rpm-integrated/amazon-ecs-volume-plugin.service amazon-ecs-volume-plugin.service
 	cp packaging/generic-rpm-integrated/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket
 	tar -czf ./sources.tgz ecs-init scripts misc agent amazon-ecs-cni-plugins amazon-vpc-cni-plugins agent-container Makefile VERSION GO_VERSION
 	test -e SOURCES || ln -s . SOURCES
-	rpmbuild --define "%version ${VERSION}" --define "%_topdir $(PWD)" -bb amazon-ecs-init.spec
+	rpmbuild --define "%_topdir $(PWD)" -bb amazon-ecs-init.spec
 	find RPMS/ -type f -exec cp {} . \;
 	touch .generic-rpm-integrated-done
 
 # Build init rpm
 generic-rpm-integrated: .generic-rpm-integrated-done
 
+VERSION = $(shell cat ecs-init/ECSVERSION)
+
 .generic-deb-integrated-done: get-cni-sources
+	./scripts/update-version.sh
 	mkdir -p BUILDROOT
 	tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md
 	cp -r packaging/generic-deb-integrated/debian Makefile ecs-init scripts misc agent agent-container amazon-ecs-cni-plugins amazon-vpc-cni-plugins README.md VERSION GO_VERSION BUILDROOT

VERSION

@@ -1 +1 @@
-1.88.0
+1.89.1

agent/api/container/container.go

@@ -1272,6 +1272,27 @@ func (c *Container) GetNetworkModeFromHostConfig() string {
 	return hostConfig.NetworkMode.NetworkName()
 }
 
+// GetMemoryReservationFromHostConfig returns the container memory reservation
+func (c *Container) GetMemoryReservationFromHostConfig() int64 {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.DockerConfig.HostConfig == nil {
+		return 0
+	}
+
+	hostConfig := &dockercontainer.HostConfig{}
+	err := json.Unmarshal([]byte(*c.DockerConfig.HostConfig), hostConfig)
+	if err != nil {
+		seelog.Warnf("Encountered error when trying to get memory reservation for container %s: %v", c.RuntimeID, err)
+		return 0
+	}
+
+	// Soft limit is specified in MiB units but translated to bytes while being transferred to Agent
+	// Converting back to MiB
+	return hostConfig.MemoryReservation / (1024 * 1024)
+}
+
 // GetHostConfig returns the container's host config.
 func (c *Container) GetHostConfig() *string {
 	c.lock.RLock()

agent/api/container/container_test.go

@@ -751,6 +751,48 @@ func TestGetNetworkModeFromHostConfig(t *testing.T) {
 	}
 }
 
+func TestGetMemoryReservationFromHostConfig(t *testing.T) {
+	getContainer := func(hostConfig *string) *Container {
+		c := &Container{
+			Name: "c",
+		}
+		c.DockerConfig.HostConfig = hostConfig
+		return c
+	}
+
+	getStrPtr := func(s string) *string {
+		return &s
+	}
+
+	testCases := []struct {
+		name           string
+		container      *Container
+		expectedOutput int64
+	}{
+		{
+			name:           "happy case",
+			container:      getContainer(getStrPtr("{\"MemoryReservation\":268435456}")),
+			expectedOutput: 256,
+		},
+		{
+			name:           "invalid case",
+			container:      getContainer(getStrPtr("invalid")),
+			expectedOutput: 0,
+		},
+		{
+			name:           "nil case",
+			container:      getContainer(nil),
+			expectedOutput: 0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expectedOutput, tc.container.GetMemoryReservationFromHostConfig())
+		})
+	}
+}
+
 func TestShouldCreateWithEnvfiles(t *testing.T) {
 	cases := []struct {
 		in  Container

agent/api/task/task.go

@@ -1246,7 +1246,7 @@ func (task *Task) initializeFirelensResource(config *config.Config, resourceFiel
 
 	for _, container := range task.Containers {
 		firelensConfig := container.GetFirelensConfig()
-		if firelensConfig != nil {
+		if firelensConfig != nil { // is Firelens container
 			var ec2InstanceID string
 			if container.Environment != nil && container.Environment[awsExecutionEnvKey] == ec2ExecutionEnv {
 				ec2InstanceID = resourceFields.EC2InstanceID
@@ -1260,9 +1260,22 @@ func (task *Task) initializeFirelensResource(config *config.Config, resourceFiel
 			} else {
 				networkMode = container.GetNetworkModeFromHostConfig()
 			}
+			// Resolve Firelens container memory limit to be used for calculating Firelens memory buffer limit
+			// We will use the container 'memoryReservation' (soft limit) if present, otherwise the container 'memory' (hard limit)
+			// If neither is present, we will pass in 0 indicating that a memory limit is not specified, and firelens will
+			// fall back to use a default memory buffer limit value
+			// see - agent/taskresource/firelens/firelensconfig_unix.go:resolveMemBufLimit()
+			var containerMemoryLimit int64
+			if memoryReservation := container.GetMemoryReservationFromHostConfig(); memoryReservation > 0 {
+				containerMemoryLimit = memoryReservation
+			} else if container.Memory > 0 {
+				containerMemoryLimit = int64(container.Memory)
+			} else {
+				containerMemoryLimit = 0
+			}
 			firelensResource, err := firelens.NewFirelensResource(config.Cluster, task.Arn, task.Family+":"+task.Version,
 				ec2InstanceID, config.DataDir, firelensConfig.Type, config.AWSRegion, networkMode, firelensConfig.Options, containerToLogOptions,
-				credentialsManager, task.ExecutionCredentialsID)
+				credentialsManager, task.ExecutionCredentialsID, containerMemoryLimit)
 			if err != nil {
 				return errors.Wrap(err, "unable to initialize firelens resource")
 			}

agent/taskresource/firelens/firelens_unimplemented.go

@@ -53,7 +53,7 @@ type FirelensResource struct{}
 // NewFirelensResource returns a new FirelensResource.
 func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDir, firelensConfigType, region, networkMode string,
 	firelensOptions map[string]string, containerToLogOptions map[string]map[string]string, credentialsManager credentials.Manager,
-	executionCredentialsID string) (*FirelensResource, error) {
+	executionCredentialsID string, containerMemoryLimit int64) (*FirelensResource, error) {
 	return nil, errors.New("not implemented")
 }
 

agent/taskresource/firelens/firelens_unix.go

@@ -83,6 +83,7 @@ type FirelensResource struct {
 	networkMode            string
 	ioutil                 ioutilwrapper.IOUtil
 	s3ClientCreator        factory.S3ClientCreator
+	containerMemoryLimit   int64
 
 	// Fields for the common functionality of task resource. Access to these fields are protected by lock.
 	createdAtUnsafe     time.Time
@@ -98,7 +99,7 @@ type FirelensResource struct {
 // NewFirelensResource returns a new FirelensResource.
 func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDir, firelensConfigType, region, networkMode string,
 	firelensOptions map[string]string, containerToLogOptions map[string]map[string]string, credentialsManager credentials.Manager,
-	executionCredentialsID string) (*FirelensResource, error) {
+	executionCredentialsID string, containerMemoryLimit int64) (*FirelensResource, error) {
 	firelensResource := &FirelensResource{
 		cluster:                cluster,
 		taskARN:                taskARN,
@@ -112,6 +113,7 @@ func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDi
 		s3ClientCreator:        factory.NewS3ClientCreator(),
 		executionCredentialsID: executionCredentialsID,
 		credentialsManager:     credentialsManager,
+		containerMemoryLimit:   containerMemoryLimit,
 	}
 
 	fields := strings.Split(taskARN, "/")