Prevent unintended resource updates to LB attatchments

docs/releases/1.19-NOTES.md

@@ -74,6 +74,20 @@ has been updated by a newer version of kops unless it is given the `--allow-kops
 
 * See note about [Openstack Cinder plugin](#openstack-cinder-plugin) above.
 
+* Terraform users, in order to prevent downtime you will have to remove the state of any existing ELB or TargetGroup attatchments from your Terraform state file. This is due to migrating the attachments to the in-line `aws_autoscaling_group` fields. See the [terraform documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) for more information about the difference. This migration is required due to a bug described in [#9913](https://github.com/kubernetes/kops/issues/9913).
+
+To prevent downtime, follow these steps with the new version of Kops:
+  ```
+  kops update cluster --target terraform ...
+  terraform plan
+  terraform state list | grep aws_autoscaling_attachment | xargs -L1 terraform state rm
+  terraform plan
+  # Ensure these resources are no longer being destroyed and recreated
+  terraform apply
+  ```
+
+* If you are using Terraform with an additional .tf file and using "aws_autoscaling_attachment" to attach additional Load Balancers or ALB/NLB Target Groups you'll need to migrate to [attaching them through the InstanceGroup spec instead](https://kops.sigs.k8s.io/instance_groups/#externalloadbalancers).
+
 # Deprecations
 
 * Support for Kubernetes versions 1.11 and 1.12 are deprecated and will be removed in kops 1.20.

pkg/model/awsmodel/api_loadbalancer.go

@@ -25,7 +25,6 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/dns"
-	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 )
@@ -273,20 +272,6 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 		elb.ForAPIServer = true
 	}
 
-	// When Spotinst Elastigroups are used, there is no need to create
-	// a separate task for the attachment of the load balancer since this
-	// is already done as part of the Elastigroup's creation, if needed.
-	if !featureflag.Spotinst.Enabled() {
-		for _, ig := range b.MasterInstanceGroups() {
-			c.AddTask(&awstasks.LoadBalancerAttachment{
-				Name:             fi.String("api-" + ig.ObjectMeta.Name),
-				Lifecycle:        b.Lifecycle,
-				AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
-				LoadBalancer:     b.LinkToELB("api"),
-			})
-		}
-	}
-
 	return nil
 
 }

pkg/model/awsmodel/autoscalinggroup.go

@@ -91,10 +91,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		c.AddTask(tsk)
 
-		// @step: add any external load balancer attachments
-		if err := b.buildExternalLoadBalancerTasks(c, ig); err != nil {
-			return err
-		}
 	}
 
 	return nil
@@ -353,6 +349,40 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
 
 	t.InstanceProtection = ig.Spec.InstanceProtection
 
+	// When Spotinst Elastigroups are used, there is no need to create
+	// a separate task for the attachment of the load balancer since this
+	// is already done as part of the Elastigroup's creation, if needed.
+	if !featureflag.Spotinst.Enabled() {
+		if b.UseLoadBalancerForAPI() && ig.Spec.Role == kops.InstanceGroupRoleMaster {
+			t.LoadBalancers = append(t.LoadBalancers, b.LinkToELB("api"))
+		}
+
+		if ig.Spec.Role == kops.InstanceGroupRoleBastion {
+			t.LoadBalancers = append(t.LoadBalancers, b.LinkToELB("bastion"))
+		}
+	}
+
+	for _, extLB := range ig.Spec.ExternalLoadBalancers {
+		if extLB.LoadBalancerName != nil {
+			t.LoadBalancers = append(t.LoadBalancers, &awstasks.LoadBalancer{Name: extLB.LoadBalancerName})
+
+			c.AddTask(&awstasks.LoadBalancer{
+				Name:   extLB.LoadBalancerName,
+				Shared: fi.Bool(true),
+			})
+		}
+
+		if extLB.TargetGroupARN != nil {
+			t.TargetGroups = append(t.TargetGroups, &awstasks.TargetGroup{Name: extLB.TargetGroupARN, ARN: extLB.TargetGroupARN})
+
+			c.AddTask(&awstasks.TargetGroup{
+				Name:   extLB.TargetGroupARN,
+				ARN:    extLB.TargetGroupARN,
+				Shared: fi.Bool(true),
+			})
+		}
+	}
+
 	// @step: are we using a mixed instance policy
 	if ig.Spec.MixedInstancesPolicy != nil {
 		spec := ig.Spec.MixedInstancesPolicy
@@ -368,28 +398,3 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
 
 	return t, nil
 }
-
-// buildExternlLoadBalancerTasks is responsible for adding any ELB attachment tasks to the model
-func (b *AutoscalingGroupModelBuilder) buildExternalLoadBalancerTasks(c *fi.ModelBuilderContext, ig *kops.InstanceGroup) error {
-	for _, x := range ig.Spec.ExternalLoadBalancers {
-		if x.LoadBalancerName != nil {
-			c.AddTask(&awstasks.ExternalLoadBalancerAttachment{
-				Name:             fi.String("extlb-" + *x.LoadBalancerName + "-" + ig.Name),
-				Lifecycle:        b.Lifecycle,
-				LoadBalancerName: *x.LoadBalancerName,
-				AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
-			})
-		}
-
-		if x.TargetGroupARN != nil {
-			c.AddTask(&awstasks.ExternalTargetGroupAttachment{
-				Name:             fi.String("exttg-" + *x.TargetGroupARN + "-" + ig.Name),
-				Lifecycle:        b.Lifecycle,
-				TargetGroupARN:   *x.TargetGroupARN,
-				AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
-			})
-		}
-	}
-
-	return nil
-}

pkg/model/bastion.go

@@ -21,7 +21,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 )
@@ -263,25 +262,6 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(elb)
 	}
 
-	// When Spotinst Elastigroups are used, there is no need to create
-	// a separate task for the attachment of the load balancer since this
-	// is already done as part of the Elastigroup's creation, if needed.
-	if !featureflag.Spotinst.Enabled() {
-		for _, ig := range bastionInstanceGroups {
-			// We build the ASG when we iterate over the instance groups
-
-			// Attach the ELB to the ASG
-			t := &awstasks.LoadBalancerAttachment{
-				Name:      s("bastion-elb-attachment"),
-				Lifecycle: b.Lifecycle,
-
-				LoadBalancer:     elb,
-				AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
-			}
-			c.AddTask(t)
-		}
-	}
-
 	bastionPublicName := ""
 	if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Bastion != nil {
 		bastionPublicName = b.Cluster.Spec.Topology.Bastion.BastionPublicName

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -110,22 +110,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "bastion-bastionuserdata-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.bastion-bastionuserdata-example-com.id
-  elb                    = aws_elb.bastion-bastionuserdata-example-com.id
-}
-
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-bastionuserdata-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-bastionuserdata-example-com.id
-  elb                    = aws_elb.api-bastionuserdata-example-com.id
-}
-
 resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.bastion-bastionuserdata-example-com.id
     version = aws_launch_template.bastion-bastionuserdata-example-com.latest_version
   }
+  load_balancers      = [aws_elb.bastion-bastionuserdata-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -174,6 +165,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-exam
     id      = aws_launch_template.master-us-test-1a-masters-bastionuserdata-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-bastionuserdata-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-bastionuserdata-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1

tests/integration/update_cluster/complex/kubernetes.tf

@@ -80,17 +80,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-complex-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-complex-example-com.id
-  elb                    = aws_elb.api-complex-example-com.id
-}
-
 resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.master-us-test-1a-masters-complex-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-complex-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-complex-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1

tests/integration/update_cluster/existing_sg/kubernetes.tf

@@ -90,27 +90,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-existingsg-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-existingsg-example-com.id
-  elb                    = aws_elb.api-existingsg-example-com.id
-}
-
-resource "aws_autoscaling_attachment" "master-us-test-1b-masters-existingsg-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1b-masters-existingsg-example-com.id
-  elb                    = aws_elb.api-existingsg-example-com.id
-}
-
-resource "aws_autoscaling_attachment" "master-us-test-1c-masters-existingsg-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1c-masters-existingsg-example-com.id
-  elb                    = aws_elb.api-existingsg-example-com.id
-}
-
 resource "aws_autoscaling_group" "master-us-test-1a-masters-existingsg-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.master-us-test-1a-masters-existingsg-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-existingsg-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-existingsg-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -159,6 +145,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-existingsg-example-c
     id      = aws_launch_template.master-us-test-1b-masters-existingsg-example-com.id
     version = aws_launch_template.master-us-test-1b-masters-existingsg-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-existingsg-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -207,6 +194,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-existingsg-example-c
     id      = aws_launch_template.master-us-test-1c-masters-existingsg-example-com.id
     version = aws_launch_template.master-us-test-1c-masters-existingsg-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-existingsg-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1

tests/integration/update_cluster/externallb/kubernetes.tf

@@ -80,27 +80,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "extlb-my-elb-nodes" {
-  autoscaling_group_name = aws_autoscaling_group.nodes-externallb-example-com.id
-  elb                    = "my-elb"
-}
-
-resource "aws_autoscaling_attachment" "extlb-my-other-elb-master-us-test-1a" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id
-  elb                    = "my-other-elb"
-}
-
-resource "aws_autoscaling_attachment" "exttg-aws_my-tg--0123456789abcdef-master-us-test-1a" {
-  alb_target_group_arn   = "aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id
-}
-
 resource "aws_autoscaling_group" "master-us-test-1a-masters-externallb-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.master-us-test-1a-masters-externallb-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-externallb-example-com.latest_version
   }
+  load_balancers      = ["my-other-elb"]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -140,6 +126,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-externallb-example-c
     propagate_at_launch = true
     value               = "owned"
   }
+  target_group_arns   = ["aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"]
   vpc_zone_identifier = [aws_subnet.us-test-1a-externallb-example-com.id]
 }
 
@@ -149,6 +136,7 @@ resource "aws_autoscaling_group" "nodes-externallb-example-com" {
     id      = aws_launch_template.nodes-externallb-example-com.id
     version = aws_launch_template.nodes-externallb-example-com.latest_version
   }
+  load_balancers      = ["my-elb"]
   max_size            = 2
   metrics_granularity = "1Minute"
   min_size            = 2

tests/integration/update_cluster/externalpolicies/kubernetes.tf

@@ -80,17 +80,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-externalpolicies-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-externalpolicies-example-com.id
-  elb                    = aws_elb.api-externalpolicies-example-com.id
-}
-
 resource "aws_autoscaling_group" "master-us-test-1a-masters-externalpolicies-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.master-us-test-1a-masters-externalpolicies-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-externalpolicies-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-externalpolicies-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1

tests/integration/update_cluster/private-shared-subnet/kubernetes.tf

@@ -100,22 +100,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "bastion-private-shared-subnet-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.bastion-private-shared-subnet-example-com.id
-  elb                    = aws_elb.bastion-private-shared-subnet-example-com.id
-}
-
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-private-shared-subnet-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-private-shared-subnet-example-com.id
-  elb                    = aws_elb.api-private-shared-subnet-example-com.id
-}
-
 resource "aws_autoscaling_group" "bastion-private-shared-subnet-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.bastion-private-shared-subnet-example-com.id
     version = aws_launch_template.bastion-private-shared-subnet-example-com.latest_version
   }
+  load_balancers      = [aws_elb.bastion-private-shared-subnet-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -164,6 +155,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-private-shared-subne
     id      = aws_launch_template.master-us-test-1a-masters-private-shared-subnet-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-private-shared-subnet-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-private-shared-subnet-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1

tests/integration/update_cluster/privatecalico/kubernetes.tf

@@ -110,22 +110,13 @@ provider "aws" {
   region = "us-test-1"
 }
 
-resource "aws_autoscaling_attachment" "bastion-privatecalico-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.bastion-privatecalico-example-com.id
-  elb                    = aws_elb.bastion-privatecalico-example-com.id
-}
-
-resource "aws_autoscaling_attachment" "master-us-test-1a-masters-privatecalico-example-com" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-privatecalico-example-com.id
-  elb                    = aws_elb.api-privatecalico-example-com.id
-}
-
 resource "aws_autoscaling_group" "bastion-privatecalico-example-com" {
   enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
   launch_template {
     id      = aws_launch_template.bastion-privatecalico-example-com.id
     version = aws_launch_template.bastion-privatecalico-example-com.latest_version
   }
+  load_balancers      = [aws_elb.bastion-privatecalico-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1
@@ -174,6 +165,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecalico-exampl
     id      = aws_launch_template.master-us-test-1a-masters-privatecalico-example-com.id
     version = aws_launch_template.master-us-test-1a-masters-privatecalico-example-com.latest_version
   }
+  load_balancers      = [aws_elb.api-privatecalico-example-com.id]
   max_size            = 1
   metrics_granularity = "1Minute"
   min_size            = 1