diff --git a/cloud/annotations/annotations.go b/cloud/annotations/annotations.go index 76e2d83f..197542bb 100644 --- a/cloud/annotations/annotations.go +++ b/cloud/annotations/annotations.go @@ -53,4 +53,10 @@ const ( NodeBalancerBackendVPCName = "service.beta.kubernetes.io/linode-loadbalancer-backend-vpc-name" NodeBalancerBackendSubnetName = "service.beta.kubernetes.io/linode-loadbalancer-backend-subnet-name" NodeBalancerBackendSubnetID = "service.beta.kubernetes.io/linode-loadbalancer-backend-subnet-id" + + NodeBalancerFrontendIPv4Range = "service.beta.kubernetes.io/linode-loadbalancer-frontend-ipv4-range" + NodeBalancerFrontendIPv6Range = "service.beta.kubernetes.io/linode-loadbalancer-frontend-ipv6-range" + NodeBalancerFrontendVPCName = "service.beta.kubernetes.io/linode-loadbalancer-frontend-vpc-name" + NodeBalancerFrontendSubnetName = "service.beta.kubernetes.io/linode-loadbalancer-frontend-subnet-name" + NodeBalancerFrontendSubnetID = "service.beta.kubernetes.io/linode-loadbalancer-frontend-subnet-id" ) diff --git a/cloud/linode/loadbalancers.go b/cloud/linode/loadbalancers.go index 681f4b75..798db4d1 100644 --- a/cloud/linode/loadbalancers.go +++ b/cloud/linode/loadbalancers.go @@ -848,6 +848,130 @@ func (l *loadbalancers) getVPCCreateOptions(ctx context.Context, service *v1.Ser return vpcCreateOpts, nil } +// getFrontendVPCCreateOptions returns the VPC options for the NodeBalancer frontend VPC creation. +// Order of precedence: +// 1. Frontend IPv4/IPv6 Range Annotations - Explicit CIDR ranges +// 2. Frontend VPC/Subnet Name Annotations - Resolve by name +// 3. Frontend Subnet ID Annotation - Direct subnet ID +func (l *loadbalancers) getFrontendVPCCreateOptions(ctx context.Context, service *v1.Service) ([]linodego.NodeBalancerVPCOptions, error) { + frontendIPv4Range, hasIPv4Range := service.GetAnnotations()[annotations.NodeBalancerFrontendIPv4Range] + frontendIPv6Range, hasIPv6Range := service.GetAnnotations()[annotations.NodeBalancerFrontendIPv6Range] + _, hasVPCName := service.GetAnnotations()[annotations.NodeBalancerFrontendVPCName] + _, hasSubnetName := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetName] + _, hasSubnetID := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetID] + + // If no frontend VPC annotations are present, return empty slice + if !hasIPv4Range && !hasIPv6Range && !hasVPCName && !hasSubnetName && !hasSubnetID { + return nil, nil + } + + var subnetID int + var err error + + // Precedence 1: IPv4/IPv6 Range Annotations - Explicit CIDR ranges + if hasIPv4Range || hasIPv6Range { + if err = validateNodeBalancerFrontendIPv4Range(frontendIPv4Range); err != nil { + return nil, err + } + if err = validateNodeBalancerFrontendIPv6Range(frontendIPv6Range); err != nil { + return nil, err + } + if frontendSubnetID, ok := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetID]; ok { + subnetID, err = strconv.Atoi(frontendSubnetID) + if err != nil { + return nil, fmt.Errorf("invalid frontend subnet ID: %w", err) + } + } else { + subnetID, err = l.getFrontendSubnetIDForSVC(ctx, service) + if err != nil { + return nil, err + } + } + + vpcCreateOpts := []linodego.NodeBalancerVPCOptions{ + { + SubnetID: subnetID, + IPv4Range: frontendIPv4Range, + IPv6Range: frontendIPv6Range, + }, + } + return vpcCreateOpts, nil + } + + // Precedence 2: VPC/Subnet Name Annotations - Resolve by name + if hasVPCName || hasSubnetName { + subnetID, err = l.getFrontendSubnetIDForSVC(ctx, service) + if err != nil { + return nil, err + } + + vpcCreateOpts := []linodego.NodeBalancerVPCOptions{ + { + SubnetID: subnetID, + }, + } + return vpcCreateOpts, nil + } + + // Precedence 3: Subnet ID Annotation - Direct subnet ID + if hasSubnetID { + frontendSubnetID := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetID] + subnetID, err = strconv.Atoi(frontendSubnetID) + if err != nil { + return nil, fmt.Errorf("invalid frontend subnet ID: %w", err) + } + + vpcCreateOpts := []linodego.NodeBalancerVPCOptions{ + { + SubnetID: subnetID, + }, + } + return vpcCreateOpts, nil + } + + return nil, nil +} + +// getFrontendSubnetIDForSVC returns the subnet ID for the frontend VPC configuration. +// Following precedence rules are applied: +// 1. If the service has an annotation for FrontendSubnetID, use that. +// 2. If the service has annotations specifying FrontendVPCName or FrontendSubnetName, use them. +// 3. Return error if no VPC configuration is found. +func (l *loadbalancers) getFrontendSubnetIDForSVC(ctx context.Context, service *v1.Service) (int, error) { + // Check if the service has an annotation for FrontendSubnetID + if specifiedSubnetID, ok := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetID]; ok { + subnetID, err := strconv.Atoi(specifiedSubnetID) + if err != nil { + return 0, fmt.Errorf("invalid frontend subnet ID: %w", err) + } + return subnetID, nil + } + + specifiedVPCName, vpcOk := service.GetAnnotations()[annotations.NodeBalancerFrontendVPCName] + specifiedSubnetName, subnetOk := service.GetAnnotations()[annotations.NodeBalancerFrontendSubnetName] + + // If no VPCName or SubnetName is specified, return error + if !vpcOk && !subnetOk { + return 0, fmt.Errorf("frontend VPC configuration requires either vpc-name, subnet-name, or subnet-id annotations") + } + + // Require both VPC name and subnet name when using name-based resolution + if !vpcOk { + return 0, fmt.Errorf("frontend VPC configuration with subnet-name requires vpc-name annotation") + } + if !subnetOk { + return 0, fmt.Errorf("frontend VPC configuration with vpc-name requires subnet-name annotation") + } + + vpcID, err := services.GetVPCID(ctx, l.client, specifiedVPCName) + if err != nil { + return 0, fmt.Errorf("failed to get VPC ID for frontend VPC '%s': %w", specifiedVPCName, err) + } + + // Use the VPC ID and Subnet Name to get the subnet ID + return services.GetSubnetID(ctx, l.client, vpcID, specifiedSubnetName) +} + func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName string, service *v1.Service, configs []*linodego.NodeBalancerConfigCreateOptions) (lb *linodego.NodeBalancer, err error) { connThrottle := getConnectionThrottle(service) @@ -870,6 +994,13 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri } } + // Add frontend VPC configuration + if frontendVPCs, err := l.getFrontendVPCCreateOptions(ctx, service); err != nil { + return nil, err + } else if len(frontendVPCs) > 0 { + createOpts.FrontendVPCs = frontendVPCs + } + // Check for static IPv4 address annotation if ipv4, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerReservedIPv4]; ok { createOpts.IPv4 = &ipv4 @@ -1336,6 +1467,12 @@ func makeLoadBalancerStatus(service *v1.Service, nb *linodego.NodeBalancer) *v1. } } + // Debug info log: Is a frontend VPC NodeBalancer? + isFrontendVPC := nb.FrontendAddressType != nil && *nb.FrontendAddressType == "vpc" + if isFrontendVPC { + klog.V(4).Infof("NodeBalancer (%d) is using frontend VPC address type", nb.ID) + } + // Check for per-service IPv6 annotation first, then fall back to global setting useIPv6 := getServiceBoolAnnotation(service, annotations.AnnLinodeEnableIPv6Ingress) || options.Options.EnableIPv6ForLoadBalancers @@ -1403,6 +1540,32 @@ func validateNodeBalancerBackendIPv4Range(backendIPv4Range string) error { return nil } +// validateNodeBalancerFrontendIPv4Range validates the frontend IPv4 range annotation. +// Performs basic CIDR format validation. +func validateNodeBalancerFrontendIPv4Range(frontendIPv4Range string) error { + if frontendIPv4Range == "" { + return nil + } + _, _, err := net.ParseCIDR(frontendIPv4Range) + if err != nil { + return fmt.Errorf("invalid frontend IPv4 range '%s': %w", frontendIPv4Range, err) + } + return nil +} + +// validateNodeBalancerFrontendIPv6Range validates the frontend IPv6 range annotation. +// Performs basic CIDR format validation. +func validateNodeBalancerFrontendIPv6Range(frontendIPv6Range string) error { + if frontendIPv6Range == "" { + return nil + } + _, _, err := net.ParseCIDR(frontendIPv6Range) + if err != nil { + return fmt.Errorf("invalid frontend IPv6 range '%s': %w", frontendIPv6Range, err) + } + return nil +} + // isCIDRWithinCIDR returns true if the inner CIDR is within the outer CIDR. func isCIDRWithinCIDR(outer, inner string) (bool, error) { _, ipNet1, err := net.ParseCIDR(outer) diff --git a/cloud/linode/loadbalancers_test.go b/cloud/linode/loadbalancers_test.go index fdb09b7c..23a3b0c5 100644 --- a/cloud/linode/loadbalancers_test.go +++ b/cloud/linode/loadbalancers_test.go @@ -18,6 +18,7 @@ import ( "testing" ciliumclient "github.com/cilium/cilium/pkg/k8s/client/clientset/versioned/typed/cilium.io/v2alpha1" + "github.com/golang/mock/gomock" "github.com/linode/linodego" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -29,6 +30,7 @@ import ( "github.com/linode/linode-cloud-controller-manager/cloud/annotations" "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" "github.com/linode/linode-cloud-controller-manager/cloud/linode/options" "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" ) @@ -5430,3 +5432,268 @@ func Test_validateNodeBalancerBackendIPv4Range(t *testing.T) { }) } } + +func Test_validateNodeBalancerFrontendIPv4Range(t *testing.T) { + type args struct { + frontendIPv4Range string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Valid IPv4 range", + args: args{frontendIPv4Range: "10.100.5.0/24"}, + wantErr: false, + }, + { + name: "Invalid IPv4 range - no CIDR", + args: args{frontendIPv4Range: "10.100.5.0"}, + wantErr: true, + }, + { + name: "Invalid IPv4 range - malformed", + args: args{frontendIPv4Range: "not-an-ip"}, + wantErr: true, + }, + { + name: "Empty range should pass", + args: args{frontendIPv4Range: ""}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := validateNodeBalancerFrontendIPv4Range(tt.args.frontendIPv4Range); (err != nil) != tt.wantErr { + t.Errorf("validateNodeBalancerFrontendIPv4Range() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_validateNodeBalancerFrontendIPv6Range(t *testing.T) { + type args struct { + frontendIPv6Range string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Valid IPv6 range", + args: args{frontendIPv6Range: "2001:db80:1005::/48"}, + wantErr: false, + }, + { + name: "Invalid IPv6 range - no CIDR", + args: args{frontendIPv6Range: "2001:db80:1005::"}, + wantErr: true, + }, + { + name: "Invalid IPv6 range - malformed", + args: args{frontendIPv6Range: "not-an-ipv6"}, + wantErr: true, + }, + { + name: "Empty range should pass", + args: args{frontendIPv6Range: ""}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := validateNodeBalancerFrontendIPv6Range(tt.args.frontendIPv6Range); (err != nil) != tt.wantErr { + t.Errorf("validateNodeBalancerFrontendIPv6Range() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_makeLoadBalancerStatus_FrontendVPC(t *testing.T) { + type args struct { + service *v1.Service + nb *linodego.NodeBalancer + } + tests := []struct { + name string + args args + want *v1.LoadBalancerStatus + }{ + { + name: "Frontend VPC NodeBalancer with IPv4", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + nb: &linodego.NodeBalancer{ + ID: 123, + Hostname: &[]string{"nb-123.example.com"}[0], + IPv4: &[]string{"10.100.5.10"}[0], + IPv6: nil, + FrontendAddressType: &[]string{"vpc"}[0], + }, + }, + want: &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: "nb-123.example.com", + IP: "10.100.5.10", + }, + }, + }, + }, + { + name: "Frontend VPC NodeBalancer with IPv4 and IPv6", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotations.AnnLinodeEnableIPv6Ingress: "true", + }, + }, + }, + nb: &linodego.NodeBalancer{ + ID: 123, + Hostname: &[]string{"nb-123.example.com"}[0], + IPv4: &[]string{"10.100.5.10"}[0], + IPv6: &[]string{"2001:db80:1005::10"}[0], + FrontendAddressType: &[]string{"vpc"}[0], + }, + }, + want: &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: "nb-123.example.com", + IP: "10.100.5.10", + }, + { + Hostname: "nb-123.example.com", + IP: "2001:db80:1005::10", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := makeLoadBalancerStatus(tt.args.service, tt.args.nb) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("makeLoadBalancerStatus() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getFrontendVPCCreateOptions(t *testing.T) { + type args struct { + service *v1.Service + } + tests := []struct { + name string + args args + want []linodego.NodeBalancerVPCOptions + wantErr bool + prepareMock func(*mocks.MockClient) + }{ + { + name: "No frontend VPC annotations", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + }, + want: nil, + wantErr: false, + }, + { + name: "Frontend IPv4 range annotation", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotations.NodeBalancerFrontendIPv4Range: "10.100.5.0/24", + annotations.NodeBalancerFrontendSubnetID: "123", + }, + }, + }, + }, + want: []linodego.NodeBalancerVPCOptions{ + { + SubnetID: 123, + IPv4Range: "10.100.5.0/24", + }, + }, + wantErr: false, + }, + { + name: "Frontend IPv6 range annotation", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotations.NodeBalancerFrontendIPv6Range: "2001:db80:1005::/48", + annotations.NodeBalancerFrontendSubnetID: "123", + }, + }, + }, + }, + want: []linodego.NodeBalancerVPCOptions{ + { + SubnetID: 123, + IPv6Range: "2001:db80:1005::/48", + }, + }, + wantErr: false, + }, + { + name: "Frontend VPC and subnet names", + args: args{ + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotations.NodeBalancerFrontendVPCName: "my-vpc", + annotations.NodeBalancerFrontendSubnetName: "frontend-subnet", + }, + }, + }, + }, + want: nil, // Will return error due to missing client setup + wantErr: true, + prepareMock: func(m *mocks.MockClient) { + m.EXPECT().ListVPCs(gomock.Any(), gomock.Any()).Return(nil, stderrors.New("mock error")) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockClient(ctrl) + if tt.prepareMock != nil { + tt.prepareMock(mockClient) + } + + l := &loadbalancers{ + client: mockClient, + } + got, err := l.getFrontendVPCCreateOptions(context.Background(), tt.args.service) + if (err != nil) != tt.wantErr { + t.Errorf("getFrontendVPCCreateOptions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + t.Errorf("getFrontendVPCCreateOptions() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cloud/linode/route_controller.go b/cloud/linode/route_controller.go index 31fc0b9d..bf785849 100644 --- a/cloud/linode/route_controller.go +++ b/cloud/linode/route_controller.go @@ -245,9 +245,8 @@ func (r *routes) DeleteRoute(ctx context.Context, clusterName string, route *clo func (r *routes) handleInterfaces(ctx context.Context, intfRoutes []string, linodeInterfaceRoutes []linodego.VPCInterfaceIPv4RangeCreateOptions, instance *linodego.Instance, intfVPCIP linodego.VPCIP, route *cloudprovider.Route) error { if instance.InterfaceGeneration == linodego.GenerationLinode { interfaceUpdateOptions := linodego.LinodeInterfaceUpdateOptions{ - VPC: &linodego.VPCInterfaceCreateOptions{ - SubnetID: intfVPCIP.SubnetID, - IPv4: &linodego.VPCInterfaceIPv4CreateOptions{Ranges: linodeInterfaceRoutes}, + VPC: &linodego.VPCInterfaceUpdateOptions{ + IPv4: &linodego.VPCInterfaceIPv4CreateOptions{Ranges: &linodeInterfaceRoutes}, }, } resp, err := r.client.UpdateInterface(ctx, instance.ID, intfVPCIP.InterfaceID, interfaceUpdateOptions) diff --git a/go.mod b/go.mod index 3ffe33a4..17af4e92 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,7 @@ require ( golang.org/x/crypto v0.45.0 // indirect golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/term v0.37.0 // indirect @@ -171,6 +171,7 @@ require ( ) replace ( + github.com/linode/linodego => github.com/komer3/linodego v0.0.0-20251201202808-852faf57b6e0 k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.34.1 k8s.io/cri-api => k8s.io/cri-api v0.34.1 k8s.io/cri-client => k8s.io/cri-client v0.34.1 diff --git a/go.sum b/go.sum index c1d5b23e..d7e5b6a9 100644 --- a/go.sum +++ b/go.sum @@ -186,6 +186,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/komer3/linodego v0.0.0-20251201202808-852faf57b6e0 h1:mlJ8Z51Aa6xG1cLbH7HjacHdb1raPQlLIJTO5s726dE= +github.com/komer3/linodego v0.0.0-20251201202808-852faf57b6e0/go.mod h1:u+mbth1igHGsd8VasP+8LKHrxuCYsVMHbY3fOYRh/FU= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -194,8 +196,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/linode/linodego v1.58.0 h1:yf/tDTf5v74qcxqWWvu3bEDOmDT1ulG9NBDdivp0/oM= -github.com/linode/linodego v1.58.0/go.mod h1:ViH3Tun41yQdknbSyrdHz/iFDXsquLu+YwFdFneEZbY= github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0= github.com/mackerelio/go-osstat v0.2.6/go.mod h1:lRy8V9ZuHpuRVZh+vyTkODeDPl3/d5MgXHtLSaqG8bA= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= @@ -393,8 +393,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=