Cluster
Provides an Elastic MapReduce Cluster, a web service that makes it easy to process large amounts of data efficiently. See Amazon Elastic MapReduce Documentation for more information. To configure Instance Groups for task nodes, see the aws.emr.InstanceGroup resource.
Example Usage
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterBootstrapActionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.releaseLabel("emr-4.6.0")
.applications("Spark")
.additionalInfo("""
{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
""")
.terminationProtection(false)
.keepJobFlowAliveWhenNoSteps(true)
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(aws_subnet.main().id())
.emrManagedMasterSecurityGroup(aws_security_group.sg().id())
.emrManagedSlaveSecurityGroup(aws_security_group.sg().id())
.instanceProfile(aws_iam_instance_profile.emr_profile().arn())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceType("m4.large")
.build())
.coreInstanceGroup(ClusterCoreInstanceGroupArgs.builder()
.instanceType("c4.large")
.instanceCount(1)
.ebsConfigs(ClusterCoreInstanceGroupEbsConfigArgs.builder()
.size("40")
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPrice("0.30")
.autoscalingPolicy("""
{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
""")
.build())
.ebsRootVolumeSize(100)
.tags(Map.ofEntries(
Map.entry("role", "rolename"),
Map.entry("env", "env")
))
.bootstrapActions(ClusterBootstrapActionArgs.builder()
.path("s3://elasticmapreduce/bootstrap-actions/run-if")
.name("runif")
.args(
"instance.isMaster=true",
"echo running on master node")
.build())
.configurationsJson("""
[
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
""")
.serviceRole(aws_iam_role.iam_emr_service_role().arn())
.build());
}
}Instance Fleet
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetLaunchSpecificationsArgs;
import com.pulumi.aws.emr.InstanceFleet;
import com.pulumi.aws.emr.InstanceFleetArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetInstanceTypeConfigArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetLaunchSpecificationsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.masterInstanceFleet(ClusterMasterInstanceFleetArgs.builder()
.instanceTypeConfigs(ClusterMasterInstanceFleetInstanceTypeConfigArgs.builder()
.instanceType("m4.xlarge")
.build())
.targetOnDemandCapacity(1)
.build())
.coreInstanceFleet(ClusterCoreInstanceFleetArgs.builder()
.instanceTypeConfigs(
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(80)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m3.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.launchSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("SWITCH_TO_ON_DEMAND")
.timeoutDurationMinutes(10)
.build())
.build())
.name("core fleet")
.targetOnDemandCapacity(2)
.targetSpotCapacity(2)
.build())
.build());
var task = new InstanceFleet("task", InstanceFleetArgs.builder()
.clusterId(example.id())
.instanceTypeConfigs(
InstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
InstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.launchSpecifications(InstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(InstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("TERMINATE_CLUSTER")
.timeoutDurationMinutes(10)
.build())
.build())
.targetOnDemandCapacity(1)
.targetSpotCapacity(1)
.build());
}
}Enable Debug Logging
Debug logging in EMR is implemented as a step. It is highly recommended that you utilize the resource options configuration with ignoreChanges if other steps are being managed outside of this provider.
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterStepArgs;
import com.pulumi.aws.emr.inputs.ClusterStepHadoopJarStepArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.steps(ClusterStepArgs.builder()
.actionOnFailure("TERMINATE_CLUSTER")
.name("Setup Hadoop Debugging")
.hadoopJarStep(ClusterStepHadoopJarStepArgs.builder()
.jar("command-runner.jar")
.args("state-pusher-script")
.build())
.build())
.build());
}
}Multiple Node Master Instance Group
Available in EMR version 5.23.0 and later, an EMR Cluster can be launched with three master nodes for high availability. Additional information about this functionality and its requirements can be found in the EMR Management Guide.
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.ec2.Subnet;
import com.pulumi.aws.ec2.SubnetArgs;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var exampleSubnet = new Subnet("exampleSubnet", SubnetArgs.builder()
.mapPublicIpOnLaunch(true)
.build());
var exampleCluster = new Cluster("exampleCluster", ClusterArgs.builder()
.releaseLabel("emr-5.24.1")
.terminationProtection(true)
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(exampleSubnet.id())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceCount(3)
.build())
.coreInstanceGroup()
.build());
}
}Import
EMR clusters can be imported using the id, e.g.,
$ pulumi import aws:emr/cluster:Cluster cluster j-123456ABCDEFSince the API does not return the actual values for Kerberos configurations, environments with those configurations will need to use the ignore_changes option available to all resources to prevent perpetual differences, e.g., terraform resource "aws_emr_cluster" "example" {
... other configuration ...
lifecycle { ignore_changes = kerberos_attributes } }
Constructors
Properties
JSON string for selecting additional features such as adding proxy information. Note: Currently there is no API to retrieve the value of this argument after EMR cluster creation from provider, therefore the provider cannot detect drift from the actual EMR cluster if its value is changed outside the provider.
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
List of configurations supplied for the EMR cluster you are creating. Supply a configuration object for applications to override their default configuration. See AWS Documentation for more information.
JSON string for supplying list of configurations for the EMR cluster.
Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any core_instance_group configuration blocks are set. Detailed below.
Configuration block to use an Instance Group for the core node type.
Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
Attributes for the EC2 instances running the job flow. See below.
Switch on/off run cluster with no steps or when all steps are complete (default is on)
Kerberos configuration for the cluster. See below.
List of step states used to filter returned steps
AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any master_instance_group configuration blocks are set. Detailed below.
Configuration block to use an Instance Group for the master node type.
The specified placement group configuration for an Amazon EMR cluster.
Release label for the Amazon EMR release.
Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized.
Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with release_label 4.8.0 or greater.
IAM role that will be assumed by the Amazon EMR service to access AWS resources. The following arguments are optional:
Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with release_label 5.28.0 or greater (default is 1).
List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with ignoreChanges if other steps are being managed outside of this provider.
Switch on/off termination protection (default is false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set to false.
Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is true.