Skip to content

Commit 8153237

Browse files
authored
fix(glue): support Ray jobs with Runtime parameter (#25867)
AWS Glue recently changed API and CloudFormation spec to require `Runtime` parameter for Ray jobs. To address the changes, I am submitting this PR. This PR is for fixing the issue #25787. ## Reference CloudFormation doc has been also updated for `Runtime` parameter. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-job-jobcommand.html#cfn-glue-job-jobcommand-runtime ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent af913b3 commit 8153237

File tree

12 files changed

+163
-100
lines changed

12 files changed

+163
-100
lines changed

packages/@aws-cdk/aws-glue-alpha/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ new glue.Job(this, 'RayJob', {
9494
executable: glue.JobExecutable.pythonRay({
9595
glueVersion: glue.GlueVersion.V4_0,
9696
pythonVersion: glue.PythonVersion.THREE_NINE,
97+
runtime: glue.Runtime.RAY_TWO_FOUR,
9798
script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),
9899
}),
99100
workerType: glue.WorkerType.Z_2X,

packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,34 @@ export enum PythonVersion {
8787
THREE_NINE = '3.9',
8888
}
8989

90+
/**
91+
* AWS Glue runtime determines the runtime engine of the job.
92+
*
93+
*/
94+
export class Runtime {
95+
/**
96+
* Runtime for a Glue for Ray 2.4.
97+
*/
98+
public static readonly RAY_TWO_FOUR = new Runtime('Ray2.4');
99+
100+
/**
101+
* Custom runtime
102+
* @param runtime custom runtime
103+
*/
104+
public static of(runtime: string): Runtime {
105+
return new Runtime(runtime);
106+
}
107+
108+
/**
109+
* The name of this Runtime.
110+
*/
111+
public readonly name: string;
112+
113+
private constructor(name: string) {
114+
this.name = name;
115+
}
116+
}
117+
90118
/**
91119
* The job type.
92120
*
@@ -150,6 +178,12 @@ interface PythonExecutableProps {
150178
}
151179

152180
interface SharedJobExecutableProps {
181+
/**
182+
* Runtime. It is required for Ray jobs.
183+
*
184+
*/
185+
readonly runtime?: Runtime;
186+
153187
/**
154188
* Glue version.
155189
*
@@ -347,6 +381,9 @@ export class JobExecutable {
347381
if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) {
348382
throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray');
349383
}
384+
if (config.runtime === undefined && config.type === JobType.RAY) {
385+
throw new Error('Runtime is required for Ray jobs.');
386+
}
350387
this.config = config;
351388
}
352389

@@ -388,6 +425,13 @@ export interface JobExecutableConfig {
388425
*/
389426
readonly pythonVersion?: PythonVersion;
390427

428+
/**
429+
* The Runtime to use.
430+
*
431+
* @default - no runtime specified
432+
*/
433+
readonly runtime?: Runtime;
434+
391435
/**
392436
* The script that is executed by a job.
393437
*/

packages/@aws-cdk/aws-glue-alpha/lib/job.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,7 @@ export class Job extends JobBase {
685685
name: executable.type.name,
686686
scriptLocation: this.codeS3ObjectUrl(executable.script),
687687
pythonVersion: executable.pythonVersion,
688+
runtime: executable.runtime ? executable.runtime.name : undefined,
688689
},
689690
glueVersion: executable.glueVersion.name,
690691
workerType: props.workerType?.name,

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "29.0.0",
2+
"version": "32.0.0",
33
"files": {
44
"432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
55
"source": {
@@ -14,15 +14,15 @@
1414
}
1515
}
1616
},
17-
"b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce": {
17+
"e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89": {
1818
"source": {
1919
"path": "aws-glue-job.template.json",
2020
"packaging": "file"
2121
},
2222
"destinations": {
2323
"current_account-current_region": {
2424
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json",
25+
"objectKey": "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
2626
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
2727
}
2828
}

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@
354354
"Tags": {
355355
"key": "value"
356356
},
357-
"WorkerType": "G.025X"
357+
"WorkerType": "G.1X"
358358
}
359359
},
360360
"EtlJob30ServiceRole8E675579": {
@@ -1415,6 +1415,7 @@
14151415
"Command": {
14161416
"Name": "glueray",
14171417
"PythonVersion": "3.9",
1418+
"Runtime": "Ray2.4",
14181419
"ScriptLocation": {
14191420
"Fn::Join": [
14201421
"",
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":"29.0.0"}
1+
{"version":"32.0.0"}

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "29.0.0",
2+
"version": "32.0.0",
33
"testCases": {
44
"integ.job": {
55
"stacks": [

packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "29.0.0",
2+
"version": "32.0.0",
33
"artifacts": {
44
"aws-glue-job.assets": {
55
"type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
1717
"validateOnSynth": false,
1818
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
1919
"cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
20-
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json",
20+
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
2121
"requiresBootstrapStackVersion": 6,
2222
"bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
2323
"additionalDependencies": [

0 commit comments

Comments
 (0)