Skip to content

Permissive network validations #2248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 16, 2021
Merged

Permissive network validations #2248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
85e9f5f
Permissive network validations
vishalbollu Jun 14, 2021
9dca907
Merge branch 'master' into permissive-network-verifications
RobertLucian Jun 15, 2021
028f0e2
Merge branch 'master' into permissive-network-verifications
RobertLucian Jun 16, 2021
b596332
Respond to PR comments
vishalbollu Jun 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
275 changes: 127 additions & 148 deletions pkg/lib/aws/servicequotas.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,6 @@ var _standardInstanceFamilies = strset.New("a", "c", "d", "h", "i", "m", "r", "t
var _knownInstanceFamilies = strset.Union(_standardInstanceFamilies, strset.New("p", "g", "inf", "x", "f", "mac"))

const (
_elasticIPsQuotaCode = "L-0263D0A3"
_internetGatewayQuotaCode = "L-A4707A72"
_natGatewayQuotaCode = "L-FE5A380F"
_vpcQuotaCode = "L-F678F1CE"
_securityGroupsQuotaCode = "L-E79EC296"
_securityGroupRulesQuotaCode = "L-0EA8095F"

// 11 inbound rules
_baseInboundRulesForNodeGroup = 11
_inboundRulesPerAZ = 8
Expand Down Expand Up @@ -159,181 +152,167 @@ func (c *Client) VerifyInstanceQuota(instances []InstanceTypeRequests) error {
return nil
}

func (c *Client) VerifyNetworkQuotas(
requiredInternetGateways int,
natGatewayRequired bool,
highlyAvailableNATGateway bool,
requiredVPCs int,
availabilityZones strset.Set,
numNodeGroups int,
longestCIDRWhiteList int) error {
quotaCodeToValueMap := map[string]int{
_elasticIPsQuotaCode: 0, // elastic IP quota code
_internetGatewayQuotaCode: 0, // internet gw quota code
_natGatewayQuotaCode: 0, // nat gw quota code
_vpcQuotaCode: 0, // vpc quota code
_securityGroupsQuotaCode: 0, // security groups quota code
_securityGroupRulesQuotaCode: 0, // security group rules quota code
}

err := c.ServiceQuotas().ListServiceQuotasPages(
&servicequotas.ListServiceQuotasInput{
ServiceCode: aws.String("ec2"),
},
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
if page == nil {
return false
}
for _, quota := range page.Quotas {
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
continue
}
if _, ok := quotaCodeToValueMap[*quota.QuotaCode]; ok {
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
func (c *Client) ListServiceQuotas(quotaCodes []string, serviceCodes []string) (map[string]int, error) {
desiredQuotaCodes := strset.New(quotaCodes...)
quotaCodeToValueMap := map[string]int{}

for _, serviceCode := range serviceCodes {
err := c.ServiceQuotas().ListServiceQuotasPages(
&servicequotas.ListServiceQuotasInput{
ServiceCode: aws.String(serviceCode),
},
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
if page == nil {
return false
}
}
return true
},
)
if err != nil {
return errors.WithStack(err)
for _, quota := range page.Quotas {
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
continue
}
if desiredQuotaCodes.Has(*quota.QuotaCode) {
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
}
}
return true
},
)
if err != nil {
return nil, errors.Wrap(err, serviceCode)
}
}

err = c.ServiceQuotas().ListServiceQuotasPages(
&servicequotas.ListServiceQuotasInput{
ServiceCode: aws.String("vpc"),
},
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
if page == nil {
return false
}
for _, quota := range page.Quotas {
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
continue
}
if _, ok := quotaCodeToValueMap[*quota.QuotaCode]; ok {
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
}
}
return true
},
)
return quotaCodeToValueMap, nil
}

func (c *Client) VerifyInternetGatewayQuota(internetGatewayQuota int, requiredInternetGateways int) error {
internetGatewaysInUse, err := c.ListInternetGateways()
if err != nil {
return errors.WithStack(err)
return err
}

// check internet GW quota
if requiredInternetGateways > 0 {
internetGatewaysInUse, err := c.ListInternetGateways()
if err != nil {
return err
}
if quotaCodeToValueMap[_internetGatewayQuotaCode]-len(internetGatewaysInUse)-requiredInternetGateways < 0 {
additionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - quotaCodeToValueMap[_internetGatewayQuotaCode]
return ErrorInternetGatewayLimitExceeded(quotaCodeToValueMap[_internetGatewayQuotaCode], additionalQuotaRequired, c.Region)
}
additionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - internetGatewayQuota

if additionalQuotaRequired > 0 {
return ErrorInternetGatewayLimitExceeded(internetGatewayQuota, additionalQuotaRequired, c.Region)
}
return nil
}

if natGatewayRequired {
// get NAT GW in use per selected AZ
natGateways, err := c.DescribeNATGateways()
if err != nil {
return err
}
subnets, err := c.DescribeSubnets()
if err != nil {
return err
func (c *Client) VerifyNATGatewayQuota(natGatewayQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {
// get NAT GW in use per selected AZ
natGateways, err := c.DescribeNATGateways()
if err != nil {
return err
}
subnets, err := c.DescribeSubnets()
if err != nil {
return err
}
azToGatewaysInUse := map[string]int{}
for _, natGateway := range natGateways {
if natGateway.SubnetId == nil {
continue
}
azToGatewaysInUse := map[string]int{}
for _, natGateway := range natGateways {
if natGateway.SubnetId == nil {
for _, subnet := range subnets {
if subnet.SubnetId == nil || subnet.AvailabilityZone == nil {
continue
}
for _, subnet := range subnets {
if subnet.SubnetId == nil || subnet.AvailabilityZone == nil {
continue
}
if !availabilityZones.Has(*subnet.AvailabilityZone) {
continue
}
if *subnet.SubnetId == *natGateway.SubnetId {
azToGatewaysInUse[*subnet.AvailabilityZone]++
}
if !availabilityZones.Has(*subnet.AvailabilityZone) {
continue
}
}
// check NAT GW quota
numOfExhaustedNATGatewayAZs := 0
azsWithQuotaDeficit := []string{}
for az, numActiveGatewaysOnAZ := range azToGatewaysInUse {
// -1 comes from the NAT gateway we require per AZ
azDeficit := quotaCodeToValueMap[_natGatewayQuotaCode] - numActiveGatewaysOnAZ - 1
if azDeficit < 0 {
numOfExhaustedNATGatewayAZs++
azsWithQuotaDeficit = append(azsWithQuotaDeficit, az)
if *subnet.SubnetId == *natGateway.SubnetId {
azToGatewaysInUse[*subnet.AvailabilityZone]++
}
}
if (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) {
return ErrorNATGatewayLimitExceeded(quotaCodeToValueMap[_natGatewayQuotaCode], 1, azsWithQuotaDeficit, c.Region)
}
// check NAT GW quota
numOfExhaustedNATGatewayAZs := 0
azsWithQuotaDeficit := []string{}
for az, numActiveGatewaysOnAZ := range azToGatewaysInUse {
// -1 comes from the NAT gateway we require per AZ
azDeficit := natGatewayQuota - numActiveGatewaysOnAZ - 1
if azDeficit < 0 {
numOfExhaustedNATGatewayAZs++
azsWithQuotaDeficit = append(azsWithQuotaDeficit, az)
}
}
if (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) {
return ErrorNATGatewayLimitExceeded(natGatewayQuota, 1, azsWithQuotaDeficit, c.Region)
}

// check EIP quota
if natGatewayRequired {
elasticIPsInUse, err := c.ListElasticIPs()
if err != nil {
return err
}
var requiredElasticIPs int
if highlyAvailableNATGateway {
requiredElasticIPs = len(availabilityZones)
} else {
requiredElasticIPs = 1
}
if quotaCodeToValueMap[_elasticIPsQuotaCode]-len(elasticIPsInUse)-requiredElasticIPs < 0 {
additionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - quotaCodeToValueMap[_elasticIPsQuotaCode]
return ErrorEIPLimitExceeded(quotaCodeToValueMap[_elasticIPsQuotaCode], additionalQuotaRequired, c.Region)
}
return nil
}

func (c *Client) VerifyEIPQuota(eipQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {
elasticIPsInUse, err := c.ListElasticIPs()
if err != nil {
return err
}
var requiredElasticIPs int
if highlyAvailableNATGateway {
requiredElasticIPs = len(availabilityZones)
} else {
requiredElasticIPs = 1
}

// check VPC quota
if requiredVPCs > 0 {
vpcs, err := c.DescribeVpcs()
if err != nil {
return err
}
if quotaCodeToValueMap[_vpcQuotaCode]-len(vpcs)-requiredVPCs < 0 {
additionalQuotaRequired := len(vpcs) + requiredVPCs - quotaCodeToValueMap[_vpcQuotaCode]
return ErrorVPCLimitExceeded(quotaCodeToValueMap[_vpcQuotaCode], additionalQuotaRequired, c.Region)
}
additionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - eipQuota

if additionalQuotaRequired > 0 {
return ErrorEIPLimitExceeded(eipQuota, additionalQuotaRequired, c.Region)
}

// check rules quota for nodegroup SGs
requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)
if requiredRulesForSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
additionalQuotaRequired := requiredRulesForSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
return nil
}

func (c *Client) VerifyVPCQuota(vpcQuota int, requiredVPCs int) error {
vpcs, err := c.DescribeVpcs()
if err != nil {
return err
}

// check rules quota for control plane SG
requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)
if requiredRulesForCPSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
additionalQuotaRequired := requiredRulesForCPSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
additionalQuotaRequired := len(vpcs) + requiredVPCs - vpcQuota

if additionalQuotaRequired > 0 {
return ErrorVPCLimitExceeded(vpcQuota, additionalQuotaRequired, c.Region)
}
return nil
}

// check security groups quota
func (c *Client) VerifySecurityGroupQuota(securifyGroupsQuota int, numNodeGroups int) error {
requiredSecurityGroups := requiredSecurityGroups(numNodeGroups)
sgs, err := c.DescribeSecurityGroups()
if err != nil {
return err
}
if quotaCodeToValueMap[_securityGroupsQuotaCode]-len(sgs)-requiredSecurityGroups < 0 {
additionalQuotaRequired := len(sgs) + requiredSecurityGroups - quotaCodeToValueMap[_securityGroupsQuotaCode]
return ErrorSecurityGroupLimitExceeded(quotaCodeToValueMap[_securityGroupsQuotaCode], additionalQuotaRequired, c.Region)

additionalQuotaRequired := len(sgs) + requiredSecurityGroups - securifyGroupsQuota

if additionalQuotaRequired > 0 {
return ErrorSecurityGroupLimitExceeded(securifyGroupsQuota, additionalQuotaRequired, c.Region)

}
return nil
}

func (c *Client) VerifySecurityGroupRulesQuota(
securifyGroupRulesQuota int,
availabilityZones strset.Set,
numNodeGroups int,
longestCIDRWhiteList int) error {

// check rules quota for nodegroup SGs
requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)
if requiredRulesForSG > securifyGroupRulesQuota {
additionalQuotaRequired := requiredRulesForSG - securifyGroupRulesQuota
return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)
}

// check rules quota for control plane SG
requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)
if requiredRulesForCPSG > securifyGroupRulesQuota {
additionalQuotaRequired := requiredRulesForCPSG - securifyGroupRulesQuota
return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)
}
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/types/clusterconfig/cluster_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1000,7 +1000,7 @@ func (cc *Config) Validate(awsClient *aws.Client) error {
requiredVPCs = 1
}
longestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList))
if err := awsClient.VerifyNetworkQuotas(1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), longestCIDRWhiteList); err != nil {
if err := VerifyNetworkQuotas(awsClient, 1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), longestCIDRWhiteList); err != nil {
// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
if !aws.IsAWSError(err) {
return err
Expand Down
Loading