Skip to content

Commit 1e4ffcd

Browse files
authored
feat(glue): Add G.4X and G.8X worker types for AWS Glue (#25637)
Adds new worker types for Glue announced in: https://aws.amazon.com/about-aws/whats-new/2023/05/aws-glue-large-instance-types-generally-available/ ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent a86b919 commit 1e4ffcd

File tree

6 files changed

+22
-7
lines changed

6 files changed

+22
-7
lines changed

packages/@aws-cdk/aws-glue-alpha/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ new glue.Job(this, 'ScalaSparkEtlJob', {
4444
className: 'com.example.HelloWorld',
4545
extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],
4646
}),
47+
workerType: glue.WorkerType.G_8X,
4748
description: 'an example Scala ETL job',
4849
});
4950
```

packages/@aws-cdk/aws-glue-alpha/lib/job.ts

+10
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ export class WorkerType {
3232
*/
3333
public static readonly G_2X = new WorkerType('G.2X');
3434

35+
/**
36+
* Each worker maps to 4 DPU (16 vCPU, 64 GB of memory, 256 GB disk), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for AWS Glue version 3.0 or later jobs.
37+
*/
38+
public static readonly G_4X = new WorkerType('G.4X');
39+
40+
/**
41+
* Each worker maps to 8 DPU (32 vCPU, 128 GB of memory, 512 GB disk), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for AWS Glue version 3.0 or later jobs.
42+
*/
43+
public static readonly G_8X = new WorkerType('G.8X');
44+
3545
/**
3646
* Each worker maps to 0.25 DPU (2 vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per worker. Suitable for low volume streaming jobs.
3747
*/

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@
190190
"key": "value"
191191
},
192192
"Timeout": 5,
193-
"WorkerType": "G.2X"
193+
"WorkerType": "G.1X"
194194
}
195195
},
196196
"EtlJob20SuccessMetricRule1759F889": {
@@ -547,7 +547,7 @@
547547
"key": "value"
548548
},
549549
"Timeout": 5,
550-
"WorkerType": "G.2X"
550+
"WorkerType": "G.1X"
551551
}
552552
},
553553
"EtlJob30SuccessMetricRuleF8870F8A": {
@@ -904,7 +904,7 @@
904904
"key": "value"
905905
},
906906
"Timeout": 5,
907-
"WorkerType": "G.2X"
907+
"WorkerType": "G.1X"
908908
}
909909
},
910910
"EtlJob40SuccessMetricRule00D3EF34": {

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@
292292
"key": "value"
293293
},
294294
"timeout": 5,
295-
"workerType": "G.2X"
295+
"workerType": "G.1X"
296296
}
297297
},
298298
"constructInfo": {
@@ -808,7 +808,7 @@
808808
"key": "value"
809809
},
810810
"timeout": 5,
811-
"workerType": "G.2X"
811+
"workerType": "G.1X"
812812
}
813813
},
814814
"constructInfo": {
@@ -1324,7 +1324,7 @@
13241324
"key": "value"
13251325
},
13261326
"timeout": 5,
1327-
"workerType": "G.2X"
1327+
"workerType": "G.1X"
13281328
}
13291329
},
13301330
"constructInfo": {

packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.
3131
glueVersion,
3232
script,
3333
}),
34-
workerType: glue.WorkerType.G_2X,
34+
workerType: glue.WorkerType.G_1X,
3535
workerCount: 10,
3636
maxConcurrentRuns: 2,
3737
maxRetries: 2,

packages/@aws-cdk/aws-glue-alpha/test/job.test.ts

+4
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ describe('WorkerType', () => {
1414

1515
test('.G_2X should set the name correctly', () => expect(glue.WorkerType.G_2X.name).toEqual('G.2X'));
1616

17+
test('.G_4X should set the name correctly', () => expect(glue.WorkerType.G_4X.name).toEqual('G.4X'));
18+
19+
test('.G_8X should set the name correctly', () => expect(glue.WorkerType.G_8X.name).toEqual('G.8X'));
20+
1721
test('.G_025X should set the name correctly', () => expect(glue.WorkerType.G_025X.name).toEqual('G.025X'));
1822

1923
test('.Z_2X should set the name correctly', () => expect(glue.WorkerType.Z_2X.name).toEqual('Z.2X'));

0 commit comments

Comments
 (0)