Skip to content

Commit 9d4f595

Browse files
vishalbolluvishalbollu
authored
Permissive network validations (#2248)
1 parent c21f53b commit 9d4f595

File tree

3 files changed

+262
-149
lines changed

3 files changed

+262
-149
lines changed

‎pkg/lib/aws/servicequotas.go

+127-148
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,6 @@ var _standardInstanceFamilies = strset.New("a", "c", "d", "h", "i", "m", "r", "t
3030
var _knownInstanceFamilies = strset.Union(_standardInstanceFamilies, strset.New("p", "g", "inf", "x", "f", "mac"))
3131

3232
const (
33-
_elasticIPsQuotaCode = "L-0263D0A3"
34-
_internetGatewayQuotaCode = "L-A4707A72"
35-
_natGatewayQuotaCode = "L-FE5A380F"
36-
_vpcQuotaCode = "L-F678F1CE"
37-
_securityGroupsQuotaCode = "L-E79EC296"
38-
_securityGroupRulesQuotaCode = "L-0EA8095F"
39-
4033
// 11 inbound rules
4134
_baseInboundRulesForNodeGroup = 11
4235
_inboundRulesPerAZ = 8
@@ -159,181 +152,167 @@ func (c *Client) VerifyInstanceQuota(instances []InstanceTypeRequests) error {
159152
return nil
160153
}
161154

162-
func (c *Client) VerifyNetworkQuotas(
163-
requiredInternetGateways int,
164-
natGatewayRequired bool,
165-
highlyAvailableNATGateway bool,
166-
requiredVPCs int,
167-
availabilityZones strset.Set,
168-
numNodeGroups int,
169-
longestCIDRWhiteList int) error {
170-
quotaCodeToValueMap := map[string]int{
171-
_elasticIPsQuotaCode: 0, // elastic IP quota code
172-
_internetGatewayQuotaCode: 0, // internet gw quota code
173-
_natGatewayQuotaCode: 0, // nat gw quota code
174-
_vpcQuotaCode: 0, // vpc quota code
175-
_securityGroupsQuotaCode: 0, // security groups quota code
176-
_securityGroupRulesQuotaCode: 0, // security group rules quota code
177-
}
178-
179-
err := c.ServiceQuotas().ListServiceQuotasPages(
180-
&servicequotas.ListServiceQuotasInput{
181-
ServiceCode: aws.String("ec2"),
182-
},
183-
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
184-
if page == nil {
185-
return false
186-
}
187-
for _, quota := range page.Quotas {
188-
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
189-
continue
190-
}
191-
if _, ok := quotaCodeToValueMap[*quota.QuotaCode]; ok {
192-
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
155+
func (c *Client) ListServiceQuotas(quotaCodes []string, serviceCodes []string) (map[string]int, error) {
156+
desiredQuotaCodes := strset.New(quotaCodes...)
157+
quotaCodeToValueMap := map[string]int{}
158+
159+
for _, serviceCode := range serviceCodes {
160+
err := c.ServiceQuotas().ListServiceQuotasPages(
161+
&servicequotas.ListServiceQuotasInput{
162+
ServiceCode: aws.String(serviceCode),
163+
},
164+
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
165+
if page == nil {
193166
return false
194167
}
195-
}
196-
return true
197-
},
198-
)
199-
if err != nil {
200-
return errors.WithStack(err)
168+
for _, quota := range page.Quotas {
169+
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
170+
continue
171+
}
172+
if desiredQuotaCodes.Has(*quota.QuotaCode) {
173+
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
174+
}
175+
}
176+
return true
177+
},
178+
)
179+
if err != nil {
180+
return nil, errors.Wrap(err, serviceCode)
181+
}
201182
}
202183

203-
err = c.ServiceQuotas().ListServiceQuotasPages(
204-
&servicequotas.ListServiceQuotasInput{
205-
ServiceCode: aws.String("vpc"),
206-
},
207-
func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {
208-
if page == nil {
209-
return false
210-
}
211-
for _, quota := range page.Quotas {
212-
if quota == nil || quota.QuotaCode == nil || quota.Value == nil {
213-
continue
214-
}
215-
if _, ok := quotaCodeToValueMap[*quota.QuotaCode]; ok {
216-
quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)
217-
}
218-
}
219-
return true
220-
},
221-
)
184+
return quotaCodeToValueMap, nil
185+
}
186+
187+
func (c *Client) VerifyInternetGatewayQuota(internetGatewayQuota int, requiredInternetGateways int) error {
188+
internetGatewaysInUse, err := c.ListInternetGateways()
222189
if err != nil {
223-
return errors.WithStack(err)
190+
return err
224191
}
225192

226-
// check internet GW quota
227-
if requiredInternetGateways > 0 {
228-
internetGatewaysInUse, err := c.ListInternetGateways()
229-
if err != nil {
230-
return err
231-
}
232-
if quotaCodeToValueMap[_internetGatewayQuotaCode]-len(internetGatewaysInUse)-requiredInternetGateways < 0 {
233-
additionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - quotaCodeToValueMap[_internetGatewayQuotaCode]
234-
return ErrorInternetGatewayLimitExceeded(quotaCodeToValueMap[_internetGatewayQuotaCode], additionalQuotaRequired, c.Region)
235-
}
193+
additionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - internetGatewayQuota
194+
195+
if additionalQuotaRequired > 0 {
196+
return ErrorInternetGatewayLimitExceeded(internetGatewayQuota, additionalQuotaRequired, c.Region)
236197
}
198+
return nil
199+
}
237200

238-
if natGatewayRequired {
239-
// get NAT GW in use per selected AZ
240-
natGateways, err := c.DescribeNATGateways()
241-
if err != nil {
242-
return err
243-
}
244-
subnets, err := c.DescribeSubnets()
245-
if err != nil {
246-
return err
201+
func (c *Client) VerifyNATGatewayQuota(natGatewayQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {
202+
// get NAT GW in use per selected AZ
203+
natGateways, err := c.DescribeNATGateways()
204+
if err != nil {
205+
return err
206+
}
207+
subnets, err := c.DescribeSubnets()
208+
if err != nil {
209+
return err
210+
}
211+
azToGatewaysInUse := map[string]int{}
212+
for _, natGateway := range natGateways {
213+
if natGateway.SubnetId == nil {
214+
continue
247215
}
248-
azToGatewaysInUse := map[string]int{}
249-
for _, natGateway := range natGateways {
250-
if natGateway.SubnetId == nil {
216+
for _, subnet := range subnets {
217+
if subnet.SubnetId == nil || subnet.AvailabilityZone == nil {
251218
continue
252219
}
253-
for _, subnet := range subnets {
254-
if subnet.SubnetId == nil || subnet.AvailabilityZone == nil {
255-
continue
256-
}
257-
if !availabilityZones.Has(*subnet.AvailabilityZone) {
258-
continue
259-
}
260-
if *subnet.SubnetId == *natGateway.SubnetId {
261-
azToGatewaysInUse[*subnet.AvailabilityZone]++
262-
}
220+
if !availabilityZones.Has(*subnet.AvailabilityZone) {
221+
continue
263222
}
264-
}
265-
// check NAT GW quota
266-
numOfExhaustedNATGatewayAZs := 0
267-
azsWithQuotaDeficit := []string{}
268-
for az, numActiveGatewaysOnAZ := range azToGatewaysInUse {
269-
// -1 comes from the NAT gateway we require per AZ
270-
azDeficit := quotaCodeToValueMap[_natGatewayQuotaCode] - numActiveGatewaysOnAZ - 1
271-
if azDeficit < 0 {
272-
numOfExhaustedNATGatewayAZs++
273-
azsWithQuotaDeficit = append(azsWithQuotaDeficit, az)
223+
if *subnet.SubnetId == *natGateway.SubnetId {
224+
azToGatewaysInUse[*subnet.AvailabilityZone]++
274225
}
275226
}
276-
if (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) {
277-
return ErrorNATGatewayLimitExceeded(quotaCodeToValueMap[_natGatewayQuotaCode], 1, azsWithQuotaDeficit, c.Region)
227+
}
228+
// check NAT GW quota
229+
numOfExhaustedNATGatewayAZs := 0
230+
azsWithQuotaDeficit := []string{}
231+
for az, numActiveGatewaysOnAZ := range azToGatewaysInUse {
232+
// -1 comes from the NAT gateway we require per AZ
233+
azDeficit := natGatewayQuota - numActiveGatewaysOnAZ - 1
234+
if azDeficit < 0 {
235+
numOfExhaustedNATGatewayAZs++
236+
azsWithQuotaDeficit = append(azsWithQuotaDeficit, az)
278237
}
279238
}
239+
if (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) {
240+
return ErrorNATGatewayLimitExceeded(natGatewayQuota, 1, azsWithQuotaDeficit, c.Region)
241+
}
280242

281-
// check EIP quota
282-
if natGatewayRequired {
283-
elasticIPsInUse, err := c.ListElasticIPs()
284-
if err != nil {
285-
return err
286-
}
287-
var requiredElasticIPs int
288-
if highlyAvailableNATGateway {
289-
requiredElasticIPs = len(availabilityZones)
290-
} else {
291-
requiredElasticIPs = 1
292-
}
293-
if quotaCodeToValueMap[_elasticIPsQuotaCode]-len(elasticIPsInUse)-requiredElasticIPs < 0 {
294-
additionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - quotaCodeToValueMap[_elasticIPsQuotaCode]
295-
return ErrorEIPLimitExceeded(quotaCodeToValueMap[_elasticIPsQuotaCode], additionalQuotaRequired, c.Region)
296-
}
243+
return nil
244+
}
245+
246+
func (c *Client) VerifyEIPQuota(eipQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {
247+
elasticIPsInUse, err := c.ListElasticIPs()
248+
if err != nil {
249+
return err
250+
}
251+
var requiredElasticIPs int
252+
if highlyAvailableNATGateway {
253+
requiredElasticIPs = len(availabilityZones)
254+
} else {
255+
requiredElasticIPs = 1
297256
}
298257

299-
// check VPC quota
300-
if requiredVPCs > 0 {
301-
vpcs, err := c.DescribeVpcs()
302-
if err != nil {
303-
return err
304-
}
305-
if quotaCodeToValueMap[_vpcQuotaCode]-len(vpcs)-requiredVPCs < 0 {
306-
additionalQuotaRequired := len(vpcs) + requiredVPCs - quotaCodeToValueMap[_vpcQuotaCode]
307-
return ErrorVPCLimitExceeded(quotaCodeToValueMap[_vpcQuotaCode], additionalQuotaRequired, c.Region)
308-
}
258+
additionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - eipQuota
259+
260+
if additionalQuotaRequired > 0 {
261+
return ErrorEIPLimitExceeded(eipQuota, additionalQuotaRequired, c.Region)
309262
}
310263

311-
// check rules quota for nodegroup SGs
312-
requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)
313-
if requiredRulesForSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
314-
additionalQuotaRequired := requiredRulesForSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
315-
return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
264+
return nil
265+
}
266+
267+
func (c *Client) VerifyVPCQuota(vpcQuota int, requiredVPCs int) error {
268+
vpcs, err := c.DescribeVpcs()
269+
if err != nil {
270+
return err
316271
}
317272

318-
// check rules quota for control plane SG
319-
requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)
320-
if requiredRulesForCPSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
321-
additionalQuotaRequired := requiredRulesForCPSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
322-
return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
273+
additionalQuotaRequired := len(vpcs) + requiredVPCs - vpcQuota
274+
275+
if additionalQuotaRequired > 0 {
276+
return ErrorVPCLimitExceeded(vpcQuota, additionalQuotaRequired, c.Region)
323277
}
278+
return nil
279+
}
324280

325-
// check security groups quota
281+
func (c *Client) VerifySecurityGroupQuota(securifyGroupsQuota int, numNodeGroups int) error {
326282
requiredSecurityGroups := requiredSecurityGroups(numNodeGroups)
327283
sgs, err := c.DescribeSecurityGroups()
328284
if err != nil {
329285
return err
330286
}
331-
if quotaCodeToValueMap[_securityGroupsQuotaCode]-len(sgs)-requiredSecurityGroups < 0 {
332-
additionalQuotaRequired := len(sgs) + requiredSecurityGroups - quotaCodeToValueMap[_securityGroupsQuotaCode]
333-
return ErrorSecurityGroupLimitExceeded(quotaCodeToValueMap[_securityGroupsQuotaCode], additionalQuotaRequired, c.Region)
334287

288+
additionalQuotaRequired := len(sgs) + requiredSecurityGroups - securifyGroupsQuota
289+
290+
if additionalQuotaRequired > 0 {
291+
return ErrorSecurityGroupLimitExceeded(securifyGroupsQuota, additionalQuotaRequired, c.Region)
292+
293+
}
294+
return nil
295+
}
296+
297+
func (c *Client) VerifySecurityGroupRulesQuota(
298+
securifyGroupRulesQuota int,
299+
availabilityZones strset.Set,
300+
numNodeGroups int,
301+
longestCIDRWhiteList int) error {
302+
303+
// check rules quota for nodegroup SGs
304+
requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)
305+
if requiredRulesForSG > securifyGroupRulesQuota {
306+
additionalQuotaRequired := requiredRulesForSG - securifyGroupRulesQuota
307+
return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)
335308
}
336309

310+
// check rules quota for control plane SG
311+
requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)
312+
if requiredRulesForCPSG > securifyGroupRulesQuota {
313+
additionalQuotaRequired := requiredRulesForCPSG - securifyGroupRulesQuota
314+
return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)
315+
}
337316
return nil
338317
}
339318

‎pkg/types/clusterconfig/cluster_config.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1000,7 +1000,7 @@ func (cc *Config) Validate(awsClient *aws.Client) error {
10001000
requiredVPCs = 1
10011001
}
10021002
longestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList))
1003-
if err := awsClient.VerifyNetworkQuotas(1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), longestCIDRWhiteList); err != nil {
1003+
if err := VerifyNetworkQuotas(awsClient, 1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), longestCIDRWhiteList); err != nil {
10041004
// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
10051005
if !aws.IsAWSError(err) {
10061006
return err

0 commit comments

Comments
 (0)