pulumi-aws-kotlin/com.pulumi.aws.glue.kotlin/Job

Job

class Job : KotlinCustomResource

Provides a Glue Job resource.

Glue functionality, such as monitoring and logging of jobs, is typically managed with the default_arguments argument. See the Special Parameters Used by AWS Glue topic in the Glue developer guide for additional information.

Example Usage

Python Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    roleArn: exampleAwsIamRole.arn,
    command: {
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    role_arn=example_aws_iam_role["arn"],
    command=aws.glue.JobCommandArgs(
        script_location=f"s3://{example_aws_s3_bucket['bucket']}/example.py",
    ))
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:    pulumi.String("example"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				ScriptLocation: pulumi.String(fmt.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket)),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      roleArn: ${exampleAwsIamRole.arn}
      command:
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
Content copied to clipboard

Ray Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    roleArn: exampleAwsIamRole.arn,
    glueVersion: "4.0",
    workerType: "Z.2X",
    command: {
        name: "glueray",
        pythonVersion: "3.9",
        runtime: "Ray2.4",
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    role_arn=example_aws_iam_role["arn"],
    glue_version="4.0",
    worker_type="Z.2X",
    command=aws.glue.JobCommandArgs(
        name="glueray",
        python_version="3.9",
        runtime="Ray2.4",
        script_location=f"s3://{example_aws_s3_bucket['bucket']}/example.py",
    ))
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        RoleArn = exampleAwsIamRole.Arn,
        GlueVersion = "4.0",
        WorkerType = "Z.2X",
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            Name = "glueray",
            PythonVersion = "3.9",
            Runtime = "Ray2.4",
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:        pulumi.String("example"),
			RoleArn:     pulumi.Any(exampleAwsIamRole.Arn),
			GlueVersion: pulumi.String("4.0"),
			WorkerType:  pulumi.String("Z.2X"),
			Command: &glue.JobCommandArgs{
				Name:           pulumi.String("glueray"),
				PythonVersion:  pulumi.String("3.9"),
				Runtime:        pulumi.String("Ray2.4"),
				ScriptLocation: pulumi.String(fmt.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket)),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .roleArn(exampleAwsIamRole.arn())
            .glueVersion("4.0")
            .workerType("Z.2X")
            .command(JobCommandArgs.builder()
                .name("glueray")
                .pythonVersion("3.9")
                .runtime("Ray2.4")
                .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      roleArn: ${exampleAwsIamRole.arn}
      glueVersion: '4.0'
      workerType: Z.2X
      command:
        name: glueray
        pythonVersion: '3.9'
        runtime: Ray2.4
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
Content copied to clipboard

Scala Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    roleArn: exampleAwsIamRole.arn,
    command: {
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.scala`,
    },
    defaultArguments: {
        "--job-language": "scala",
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    role_arn=example_aws_iam_role["arn"],
    command=aws.glue.JobCommandArgs(
        script_location=f"s3://{example_aws_s3_bucket['bucket']}/example.scala",
    ),
    default_arguments={
        "--job-language": "scala",
    })
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.scala",
        },
        DefaultArguments =
        {
            { "--job-language", "scala" },
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:    pulumi.String("example"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				ScriptLocation: pulumi.String(fmt.Sprintf("s3://%v/example.scala", exampleAwsS3Bucket.Bucket)),
			},
			DefaultArguments: pulumi.StringMap{
				"--job-language": pulumi.String("scala"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .scriptLocation(String.format("s3://%s/example.scala", exampleAwsS3Bucket.bucket()))
                .build())
            .defaultArguments(Map.of("--job-language", "scala"))
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      roleArn: ${exampleAwsIamRole.arn}
      command:
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.scala
      defaultArguments:
        --job-language: scala
Content copied to clipboard

Streaming Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example streaming job",
    roleArn: exampleAwsIamRole.arn,
    command: {
        name: "gluestreaming",
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.script`,
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example streaming job",
    role_arn=example_aws_iam_role["arn"],
    command=aws.glue.JobCommandArgs(
        name="gluestreaming",
        script_location=f"s3://{example_aws_s3_bucket['bucket']}/example.script",
    ))
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example streaming job",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            Name = "gluestreaming",
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.script",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:    pulumi.String("example streaming job"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				Name:           pulumi.String("gluestreaming"),
				ScriptLocation: pulumi.String(fmt.Sprintf("s3://%v/example.script", exampleAwsS3Bucket.Bucket)),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example streaming job")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .name("gluestreaming")
                .scriptLocation(String.format("s3://%s/example.script", exampleAwsS3Bucket.bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example streaming job
      roleArn: ${exampleAwsIamRole.arn}
      command:
        name: gluestreaming
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.script
Content copied to clipboard

Enabling CloudWatch Logs and Metrics

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.cloudwatch.LogGroup("example", {
    name: "example",
    retentionInDays: 14,
});
const exampleJob = new aws.glue.Job("example", {defaultArguments: {
    "--continuous-log-logGroup": example.name,
    "--enable-continuous-cloudwatch-log": "true",
    "--enable-continuous-log-filter": "true",
    "--enable-metrics": "",
}});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.cloudwatch.LogGroup("example",
    name="example",
    retention_in_days=14)
example_job = aws.glue.Job("example", default_arguments={
    "--continuous-log-logGroup": example.name,
    "--enable-continuous-cloudwatch-log": "true",
    "--enable-continuous-log-filter": "true",
    "--enable-metrics": "",
})
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.CloudWatch.LogGroup("example", new()
    {
        Name = "example",
        RetentionInDays = 14,
    });
    var exampleJob = new Aws.Glue.Job("example", new()
    {
        DefaultArguments =
        {
            { "--continuous-log-logGroup", example.Name },
            { "--enable-continuous-cloudwatch-log", "true" },
            { "--enable-continuous-log-filter", "true" },
            { "--enable-metrics", "" },
        },
    });
});
Content copied to clipboard
package main
import (
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/cloudwatch"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		example, err := cloudwatch.NewLogGroup(ctx, "example", &cloudwatch.LogGroupArgs{
			Name:            pulumi.String("example"),
			RetentionInDays: pulumi.Int(14),
		})
		if err != nil {
			return err
		}
		_, err = glue.NewJob(ctx, "example", &glue.JobArgs{
			DefaultArguments: pulumi.StringMap{
				"--continuous-log-logGroup":          example.Name,
				"--enable-continuous-cloudwatch-log": pulumi.String("true"),
				"--enable-continuous-log-filter":     pulumi.String("true"),
				"--enable-metrics":                   pulumi.String(""),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.cloudwatch.LogGroup;
import com.pulumi.aws.cloudwatch.LogGroupArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new LogGroup("example", LogGroupArgs.builder()
            .name("example")
            .retentionInDays(14)
            .build());
        var exampleJob = new Job("exampleJob", JobArgs.builder()
            .defaultArguments(Map.ofEntries(
                Map.entry("--continuous-log-logGroup", example.name()),
                Map.entry("--enable-continuous-cloudwatch-log", "true"),
                Map.entry("--enable-continuous-log-filter", "true"),
                Map.entry("--enable-metrics", "")
            ))
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:cloudwatch:LogGroup
    properties:
      name: example
      retentionInDays: 14
  exampleJob:
    type: aws:glue:Job
    name: example
    properties:
      defaultArguments:
        --continuous-log-logGroup: ${example.name}
        --enable-continuous-cloudwatch-log: 'true'
        --enable-continuous-log-filter: 'true'
        --enable-metrics:
Content copied to clipboard

Import

Using pulumi import, import Glue Jobs using name. For example:

$ pulumi import aws:glue/job:Job MyJob MyJob
Content copied to clipboard

Properties

Link copied to clipboard
val arn: Output<String>

Amazon Resource Name (ARN) of Glue Job

Link copied to clipboard
val command: Output<JobCommand>

The command of the job. Defined below.

Link copied to clipboard
val connections: Output<List<String>>?

The list of connections used for this job.

Link copied to clipboard
val defaultArguments: Output<Map<String, String>>?

The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.

Link copied to clipboard
val description: Output<String>?

Description of the job.

Link copied to clipboard
val executionClass: Output<String>?

Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: FLEX, STANDARD.

Link copied to clipboard
val executionProperty: Output<JobExecutionProperty>

Execution property of the job. Defined below.

Link copied to clipboard
val glueVersion: Output<String>

The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.

Link copied to clipboard
val id: Output<String>
Link copied to clipboard
val maxCapacity: Output<Double>

The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. Required when pythonshell is set, accept either 0.0625 or 1.0. Use number_of_workers and worker_type arguments instead with glue_version 2.0 and above.

Link copied to clipboard
val maxRetries: Output<Int>?

The maximum number of times to retry this job if it fails.

Link copied to clipboard
val name: Output<String>

The name you assign to this job. It must be unique in your account.

Link copied to clipboard
val nonOverridableArguments: Output<Map<String, String>>?

Non-overridable arguments for this job, specified as name-value pairs.

Link copied to clipboard
val notificationProperty: Output<JobNotificationProperty>

Notification property of the job. Defined below.

Link copied to clipboard
val numberOfWorkers: Output<Int>

The number of workers of a defined workerType that are allocated when a job runs.

Link copied to clipboard
val pulumiChildResources: Set<KotlinResource>
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
val roleArn: Output<String>

The ARN of the IAM role associated with this job.

Link copied to clipboard
val securityConfiguration: Output<String>?

The name of the Security Configuration to be associated with the job.

Link copied to clipboard
val tags: Output<Map<String, String>>?

Key-value map of resource tags. If configured with a provider default_tags configuration block present, tags with matching keys will overwrite those defined at the provider-level.

Link copied to clipboard
val tagsAll: Output<Map<String, String>>

A map of tags assigned to the resource, including those inherited from the provider default_tags configuration block.

Link copied to clipboard
val timeout: Output<Int>

The job timeout in minutes. The default is 2880 minutes (48 hours) for glueetl and pythonshell jobs, and null (unlimited) for gluestreaming jobs.

Link copied to clipboard
val urn: Output<String>
Link copied to clipboard
val workerType: Output<String>

The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.