pulumi-aws-kotlin/com.pulumi.aws.glue.kotlin/JobArgs

JobArgs

data class JobArgs(val command: Output<JobCommandArgs>? = null, val connections: Output<List<String>>? = null, val defaultArguments: Output<Map<String, String>>? = null, val description: Output<String>? = null, val executionClass: Output<String>? = null, val executionProperty: Output<JobExecutionPropertyArgs>? = null, val glueVersion: Output<String>? = null, val jobRunQueuingEnabled: Output<Boolean>? = null, val maintenanceWindow: Output<String>? = null, val maxCapacity: Output<Double>? = null, val maxRetries: Output<Int>? = null, val name: Output<String>? = null, val nonOverridableArguments: Output<Map<String, String>>? = null, val notificationProperty: Output<JobNotificationPropertyArgs>? = null, val numberOfWorkers: Output<Int>? = null, val roleArn: Output<String>? = null, val securityConfiguration: Output<String>? = null, val sourceControlDetails: Output<JobSourceControlDetailsArgs>? = null, val tags: Output<Map<String, String>>? = null, val timeout: Output<Int>? = null, val workerType: Output<String>? = null) : ConvertibleToJava<JobArgs>

Provides a Glue Job resource.

Glue functionality, such as monitoring and logging of jobs, is typically managed with the default_arguments argument. See the Special Parameters Used by AWS Glue topic in the Glue developer guide for additional information.

Example Usage

Python Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    glueVersion: "5.0",
    roleArn: exampleAwsIamRole.arn,
    command: {
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
        pythonVersion: "3",
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    glue_version="5.0",
    role_arn=example_aws_iam_role["arn"],
    command={
        "script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.py",
        "python_version": "3",
    })
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        GlueVersion = "5.0",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
            PythonVersion = "3",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:        pulumi.String("example"),
			GlueVersion: pulumi.String("5.0"),
			RoleArn:     pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				ScriptLocation: pulumi.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket),
				PythonVersion:  pulumi.String("3"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .glueVersion("5.0")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
                .pythonVersion("3")
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      glueVersion: '5.0'
      roleArn: ${exampleAwsIamRole.arn}
      command:
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
        pythonVersion: '3'
Content copied to clipboard

Ray Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    roleArn: exampleAwsIamRole.arn,
    glueVersion: "4.0",
    workerType: "Z.2X",
    command: {
        name: "glueray",
        pythonVersion: "3.9",
        runtime: "Ray2.4",
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    role_arn=example_aws_iam_role["arn"],
    glue_version="4.0",
    worker_type="Z.2X",
    command={
        "name": "glueray",
        "python_version": "3.9",
        "runtime": "Ray2.4",
        "script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.py",
    })
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        RoleArn = exampleAwsIamRole.Arn,
        GlueVersion = "4.0",
        WorkerType = "Z.2X",
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            Name = "glueray",
            PythonVersion = "3.9",
            Runtime = "Ray2.4",
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:        pulumi.String("example"),
			RoleArn:     pulumi.Any(exampleAwsIamRole.Arn),
			GlueVersion: pulumi.String("4.0"),
			WorkerType:  pulumi.String("Z.2X"),
			Command: &glue.JobCommandArgs{
				Name:           pulumi.String("glueray"),
				PythonVersion:  pulumi.String("3.9"),
				Runtime:        pulumi.String("Ray2.4"),
				ScriptLocation: pulumi.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .roleArn(exampleAwsIamRole.arn())
            .glueVersion("4.0")
            .workerType("Z.2X")
            .command(JobCommandArgs.builder()
                .name("glueray")
                .pythonVersion("3.9")
                .runtime("Ray2.4")
                .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      roleArn: ${exampleAwsIamRole.arn}
      glueVersion: '4.0'
      workerType: Z.2X
      command:
        name: glueray
        pythonVersion: '3.9'
        runtime: Ray2.4
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
Content copied to clipboard

Scala Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example",
    roleArn: exampleAwsIamRole.arn,
    command: {
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.scala`,
    },
    defaultArguments: {
        "--job-language": "scala",
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example",
    role_arn=example_aws_iam_role["arn"],
    command={
        "script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.scala",
    },
    default_arguments={
        "--job-language": "scala",
    })
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.scala",
        },
        DefaultArguments =
        {
            { "--job-language", "scala" },
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:    pulumi.String("example"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				ScriptLocation: pulumi.Sprintf("s3://%v/example.scala", exampleAwsS3Bucket.Bucket),
			},
			DefaultArguments: pulumi.StringMap{
				"--job-language": pulumi.String("scala"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .scriptLocation(String.format("s3://%s/example.scala", exampleAwsS3Bucket.bucket()))
                .build())
            .defaultArguments(Map.of("--job-language", "scala"))
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example
      roleArn: ${exampleAwsIamRole.arn}
      command:
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.scala
      defaultArguments:
        --job-language: scala
Content copied to clipboard

Streaming Job

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
    name: "example streaming job",
    roleArn: exampleAwsIamRole.arn,
    command: {
        name: "gluestreaming",
        scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.script`,
    },
});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
    name="example streaming job",
    role_arn=example_aws_iam_role["arn"],
    command={
        "name": "gluestreaming",
        "script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.script",
    })
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.Glue.Job("example", new()
    {
        Name = "example streaming job",
        RoleArn = exampleAwsIamRole.Arn,
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            Name = "gluestreaming",
            ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.script",
        },
    });
});
Content copied to clipboard
package main
import (
	"fmt"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
			Name:    pulumi.String("example streaming job"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
			Command: &glue.JobCommandArgs{
				Name:           pulumi.String("gluestreaming"),
				ScriptLocation: pulumi.Sprintf("s3://%v/example.script", exampleAwsS3Bucket.Bucket),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Job("example", JobArgs.builder()
            .name("example streaming job")
            .roleArn(exampleAwsIamRole.arn())
            .command(JobCommandArgs.builder()
                .name("gluestreaming")
                .scriptLocation(String.format("s3://%s/example.script", exampleAwsS3Bucket.bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:glue:Job
    properties:
      name: example streaming job
      roleArn: ${exampleAwsIamRole.arn}
      command:
        name: gluestreaming
        scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.script
Content copied to clipboard

Enabling CloudWatch Logs and Metrics

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.cloudwatch.LogGroup("example", {
    name: "example",
    retentionInDays: 14,
});
const exampleJob = new aws.glue.Job("example", {defaultArguments: {
    "--continuous-log-logGroup": example.name,
    "--enable-continuous-cloudwatch-log": "true",
    "--enable-continuous-log-filter": "true",
    "--enable-metrics": "",
}});
Content copied to clipboard
import pulumi
import pulumi_aws as aws
example = aws.cloudwatch.LogGroup("example",
    name="example",
    retention_in_days=14)
example_job = aws.glue.Job("example", default_arguments={
    "--continuous-log-logGroup": example.name,
    "--enable-continuous-cloudwatch-log": "true",
    "--enable-continuous-log-filter": "true",
    "--enable-metrics": "",
})
Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
    var example = new Aws.CloudWatch.LogGroup("example", new()
    {
        Name = "example",
        RetentionInDays = 14,
    });
    var exampleJob = new Aws.Glue.Job("example", new()
    {
        DefaultArguments =
        {
            { "--continuous-log-logGroup", example.Name },
            { "--enable-continuous-cloudwatch-log", "true" },
            { "--enable-continuous-log-filter", "true" },
            { "--enable-metrics", "" },
        },
    });
});
Content copied to clipboard
package main
import (
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/cloudwatch"
	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		example, err := cloudwatch.NewLogGroup(ctx, "example", &cloudwatch.LogGroupArgs{
			Name:            pulumi.String("example"),
			RetentionInDays: pulumi.Int(14),
		})
		if err != nil {
			return err
		}
		_, err = glue.NewJob(ctx, "example", &glue.JobArgs{
			DefaultArguments: pulumi.StringMap{
				"--continuous-log-logGroup":          example.Name,
				"--enable-continuous-cloudwatch-log": pulumi.String("true"),
				"--enable-continuous-log-filter":     pulumi.String("true"),
				"--enable-metrics":                   pulumi.String(""),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.cloudwatch.LogGroup;
import com.pulumi.aws.cloudwatch.LogGroupArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new LogGroup("example", LogGroupArgs.builder()
            .name("example")
            .retentionInDays(14)
            .build());
        var exampleJob = new Job("exampleJob", JobArgs.builder()
            .defaultArguments(Map.ofEntries(
                Map.entry("--continuous-log-logGroup", example.name()),
                Map.entry("--enable-continuous-cloudwatch-log", "true"),
                Map.entry("--enable-continuous-log-filter", "true"),
                Map.entry("--enable-metrics", "")
            ))
            .build());
    }
}
Content copied to clipboard
resources:
  example:
    type: aws:cloudwatch:LogGroup
    properties:
      name: example
      retentionInDays: 14
  exampleJob:
    type: aws:glue:Job
    name: example
    properties:
      defaultArguments:
        --continuous-log-logGroup: ${example.name}
        --enable-continuous-cloudwatch-log: 'true'
        --enable-continuous-log-filter: 'true'
        --enable-metrics: ""
Content copied to clipboard

Import

Using pulumi import, import Glue Jobs using name. For example:

$ pulumi import aws:glue/job:Job MyJob MyJob
Content copied to clipboard

Constructors

Link copied to clipboard
constructor(command: Output<JobCommandArgs>? = null, connections: Output<List<String>>? = null, defaultArguments: Output<Map<String, String>>? = null, description: Output<String>? = null, executionClass: Output<String>? = null, executionProperty: Output<JobExecutionPropertyArgs>? = null, glueVersion: Output<String>? = null, jobRunQueuingEnabled: Output<Boolean>? = null, maintenanceWindow: Output<String>? = null, maxCapacity: Output<Double>? = null, maxRetries: Output<Int>? = null, name: Output<String>? = null, nonOverridableArguments: Output<Map<String, String>>? = null, notificationProperty: Output<JobNotificationPropertyArgs>? = null, numberOfWorkers: Output<Int>? = null, roleArn: Output<String>? = null, securityConfiguration: Output<String>? = null, sourceControlDetails: Output<JobSourceControlDetailsArgs>? = null, tags: Output<Map<String, String>>? = null, timeout: Output<Int>? = null, workerType: Output<String>? = null)

Properties

Link copied to clipboard
val command: Output<JobCommandArgs>? = null

The command of the job. Defined below.

Link copied to clipboard
val connections: Output<List<String>>? = null

The list of connections used for this job.

Link copied to clipboard
val defaultArguments: Output<Map<String, String>>? = null

The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.

Link copied to clipboard
val description: Output<String>? = null

Description of the job.

Link copied to clipboard
val executionClass: Output<String>? = null

Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: FLEX, STANDARD.

Link copied to clipboard
val executionProperty: Output<JobExecutionPropertyArgs>? = null

Execution property of the job. Defined below.

Link copied to clipboard
val glueVersion: Output<String>? = null

The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.

Link copied to clipboard
val jobRunQueuingEnabled: Output<Boolean>? = null

Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.

Link copied to clipboard
val maintenanceWindow: Output<String>? = null

Specifies the day of the week and hour for the maintenance window for streaming jobs.

Link copied to clipboard
val maxCapacity: Output<Double>? = null

The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. Required when pythonshell is set, accept either 0.0625 or 1.0. Use number_of_workers and worker_type arguments instead with glue_version 2.0 and above.

Link copied to clipboard
val maxRetries: Output<Int>? = null

The maximum number of times to retry this job if it fails.

Link copied to clipboard
val name: Output<String>? = null

The name you assign to this job. It must be unique in your account.

Link copied to clipboard
val nonOverridableArguments: Output<Map<String, String>>? = null

Non-overridable arguments for this job, specified as name-value pairs.

Link copied to clipboard
val notificationProperty: Output<JobNotificationPropertyArgs>? = null

Notification property of the job. Defined below.

Link copied to clipboard
val numberOfWorkers: Output<Int>? = null

The number of workers of a defined workerType that are allocated when a job runs.

Link copied to clipboard
val roleArn: Output<String>? = null

The ARN of the IAM role associated with this job.

Link copied to clipboard
val securityConfiguration: Output<String>? = null

The name of the Security Configuration to be associated with the job.

Link copied to clipboard
val sourceControlDetails: Output<JobSourceControlDetailsArgs>? = null

The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below.

Link copied to clipboard
val tags: Output<Map<String, String>>? = null

Key-value map of resource tags. If configured with a provider default_tags configuration block present, tags with matching keys will overwrite those defined at the provider-level.

Link copied to clipboard
val timeout: Output<Int>? = null

The job timeout in minutes. The default is 2880 minutes (48 hours) for glueetl and pythonshell jobs, and null (unlimited) for gluestreaming jobs.

Link copied to clipboard
val workerType: Output<String>? = null

The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

Functions

Link copied to clipboard
open override fun toJava(): JobArgs