Environment Args
Creates a MWAA Environment resource.
Example Usage
A MWAA Environment requires an IAM role (aws.iam.Role
), two subnets in the private zone (aws.ec2.Subnet
) and a versioned S3 bucket (aws.s3.BucketV2
).
Basic Usage
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.mwaa.Environment("example", {
dagS3Path: "dags/",
executionRoleArn: exampleAwsIamRole.arn,
name: "example",
networkConfiguration: {
securityGroupIds: [exampleAwsSecurityGroup.id],
subnetIds: _private.map(__item => __item.id),
},
sourceBucketArn: exampleAwsS3Bucket.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.mwaa.Environment("example",
dag_s3_path="dags/",
execution_role_arn=example_aws_iam_role["arn"],
name="example",
network_configuration={
"security_group_ids": [example_aws_security_group["id"]],
"subnet_ids": [__item["id"] for __item in private],
},
source_bucket_arn=example_aws_s3_bucket["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Mwaa.Environment("example", new()
{
DagS3Path = "dags/",
ExecutionRoleArn = exampleAwsIamRole.Arn,
Name = "example",
NetworkConfiguration = new Aws.Mwaa.Inputs.EnvironmentNetworkConfigurationArgs
{
SecurityGroupIds = new[]
{
exampleAwsSecurityGroup.Id,
},
SubnetIds = @private.Select(__item => __item.Id).ToList(),
},
SourceBucketArn = exampleAwsS3Bucket.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/mwaa"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := mwaa.NewEnvironment(ctx, "example", &mwaa.EnvironmentArgs{
DagS3Path: pulumi.String("dags/"),
ExecutionRoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Name: pulumi.String("example"),
NetworkConfiguration: &mwaa.EnvironmentNetworkConfigurationArgs{
SecurityGroupIds: pulumi.StringArray{
exampleAwsSecurityGroup.Id,
},
SubnetIds: []pulumi.String(%!v(PANIC=Format method: fatal: A failure has occurred: unlowered splat expression @ example.pp:6,24-37)),
},
SourceBucketArn: pulumi.Any(exampleAwsS3Bucket.Arn),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.mwaa.Environment;
import com.pulumi.aws.mwaa.EnvironmentArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentNetworkConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Environment("example", EnvironmentArgs.builder()
.dagS3Path("dags/")
.executionRoleArn(exampleAwsIamRole.arn())
.name("example")
.networkConfiguration(EnvironmentNetworkConfigurationArgs.builder()
.securityGroupIds(exampleAwsSecurityGroup.id())
.subnetIds(private_.stream().map(element -> element.id()).collect(toList()))
.build())
.sourceBucketArn(exampleAwsS3Bucket.arn())
.build());
}
}
Example with Airflow configuration options
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.mwaa.Environment("example", {
airflowConfigurationOptions: {
"core.default_task_retries": "16",
"core.parallelism": "1",
},
dagS3Path: "dags/",
executionRoleArn: exampleAwsIamRole.arn,
name: "example",
networkConfiguration: {
securityGroupIds: [exampleAwsSecurityGroup.id],
subnetIds: _private.map(__item => __item.id),
},
sourceBucketArn: exampleAwsS3Bucket.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.mwaa.Environment("example",
airflow_configuration_options={
"core.default_task_retries": "16",
"core.parallelism": "1",
},
dag_s3_path="dags/",
execution_role_arn=example_aws_iam_role["arn"],
name="example",
network_configuration={
"security_group_ids": [example_aws_security_group["id"]],
"subnet_ids": [__item["id"] for __item in private],
},
source_bucket_arn=example_aws_s3_bucket["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Mwaa.Environment("example", new()
{
AirflowConfigurationOptions =
{
{ "core.default_task_retries", "16" },
{ "core.parallelism", "1" },
},
DagS3Path = "dags/",
ExecutionRoleArn = exampleAwsIamRole.Arn,
Name = "example",
NetworkConfiguration = new Aws.Mwaa.Inputs.EnvironmentNetworkConfigurationArgs
{
SecurityGroupIds = new[]
{
exampleAwsSecurityGroup.Id,
},
SubnetIds = @private.Select(__item => __item.Id).ToList(),
},
SourceBucketArn = exampleAwsS3Bucket.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/mwaa"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := mwaa.NewEnvironment(ctx, "example", &mwaa.EnvironmentArgs{
AirflowConfigurationOptions: pulumi.StringMap{
"core.default_task_retries": pulumi.String("16"),
"core.parallelism": pulumi.String("1"),
},
DagS3Path: pulumi.String("dags/"),
ExecutionRoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Name: pulumi.String("example"),
NetworkConfiguration: &mwaa.EnvironmentNetworkConfigurationArgs{
SecurityGroupIds: pulumi.StringArray{
exampleAwsSecurityGroup.Id,
},
SubnetIds: []pulumi.String(%!v(PANIC=Format method: fatal: A failure has occurred: unlowered splat expression @ example.pp:10,24-37)),
},
SourceBucketArn: pulumi.Any(exampleAwsS3Bucket.Arn),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.mwaa.Environment;
import com.pulumi.aws.mwaa.EnvironmentArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentNetworkConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Environment("example", EnvironmentArgs.builder()
.airflowConfigurationOptions(Map.ofEntries(
Map.entry("core.default_task_retries", 16),
Map.entry("core.parallelism", 1)
))
.dagS3Path("dags/")
.executionRoleArn(exampleAwsIamRole.arn())
.name("example")
.networkConfiguration(EnvironmentNetworkConfigurationArgs.builder()
.securityGroupIds(exampleAwsSecurityGroup.id())
.subnetIds(private_.stream().map(element -> element.id()).collect(toList()))
.build())
.sourceBucketArn(exampleAwsS3Bucket.arn())
.build());
}
}
Example with logging configurations
Note that Airflow task logs are enabled by default with the INFO
log level.
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.mwaa.Environment("example", {
dagS3Path: "dags/",
executionRoleArn: exampleAwsIamRole.arn,
loggingConfiguration: {
dagProcessingLogs: {
enabled: true,
logLevel: "DEBUG",
},
schedulerLogs: {
enabled: true,
logLevel: "INFO",
},
taskLogs: {
enabled: true,
logLevel: "WARNING",
},
webserverLogs: {
enabled: true,
logLevel: "ERROR",
},
workerLogs: {
enabled: true,
logLevel: "CRITICAL",
},
},
name: "example",
networkConfiguration: {
securityGroupIds: [exampleAwsSecurityGroup.id],
subnetIds: _private.map(__item => __item.id),
},
sourceBucketArn: exampleAwsS3Bucket.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.mwaa.Environment("example",
dag_s3_path="dags/",
execution_role_arn=example_aws_iam_role["arn"],
logging_configuration={
"dag_processing_logs": {
"enabled": True,
"log_level": "DEBUG",
},
"scheduler_logs": {
"enabled": True,
"log_level": "INFO",
},
"task_logs": {
"enabled": True,
"log_level": "WARNING",
},
"webserver_logs": {
"enabled": True,
"log_level": "ERROR",
},
"worker_logs": {
"enabled": True,
"log_level": "CRITICAL",
},
},
name="example",
network_configuration={
"security_group_ids": [example_aws_security_group["id"]],
"subnet_ids": [__item["id"] for __item in private],
},
source_bucket_arn=example_aws_s3_bucket["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Mwaa.Environment("example", new()
{
DagS3Path = "dags/",
ExecutionRoleArn = exampleAwsIamRole.Arn,
LoggingConfiguration = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationArgs
{
DagProcessingLogs = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationDagProcessingLogsArgs
{
Enabled = true,
LogLevel = "DEBUG",
},
SchedulerLogs = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationSchedulerLogsArgs
{
Enabled = true,
LogLevel = "INFO",
},
TaskLogs = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationTaskLogsArgs
{
Enabled = true,
LogLevel = "WARNING",
},
WebserverLogs = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationWebserverLogsArgs
{
Enabled = true,
LogLevel = "ERROR",
},
WorkerLogs = new Aws.Mwaa.Inputs.EnvironmentLoggingConfigurationWorkerLogsArgs
{
Enabled = true,
LogLevel = "CRITICAL",
},
},
Name = "example",
NetworkConfiguration = new Aws.Mwaa.Inputs.EnvironmentNetworkConfigurationArgs
{
SecurityGroupIds = new[]
{
exampleAwsSecurityGroup.Id,
},
SubnetIds = @private.Select(__item => __item.Id).ToList(),
},
SourceBucketArn = exampleAwsS3Bucket.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/mwaa"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := mwaa.NewEnvironment(ctx, "example", &mwaa.EnvironmentArgs{
DagS3Path: pulumi.String("dags/"),
ExecutionRoleArn: pulumi.Any(exampleAwsIamRole.Arn),
LoggingConfiguration: &mwaa.EnvironmentLoggingConfigurationArgs{
DagProcessingLogs: &mwaa.EnvironmentLoggingConfigurationDagProcessingLogsArgs{
Enabled: pulumi.Bool(true),
LogLevel: pulumi.String("DEBUG"),
},
SchedulerLogs: &mwaa.EnvironmentLoggingConfigurationSchedulerLogsArgs{
Enabled: pulumi.Bool(true),
LogLevel: pulumi.String("INFO"),
},
TaskLogs: &mwaa.EnvironmentLoggingConfigurationTaskLogsArgs{
Enabled: pulumi.Bool(true),
LogLevel: pulumi.String("WARNING"),
},
WebserverLogs: &mwaa.EnvironmentLoggingConfigurationWebserverLogsArgs{
Enabled: pulumi.Bool(true),
LogLevel: pulumi.String("ERROR"),
},
WorkerLogs: &mwaa.EnvironmentLoggingConfigurationWorkerLogsArgs{
Enabled: pulumi.Bool(true),
LogLevel: pulumi.String("CRITICAL"),
},
},
Name: pulumi.String("example"),
NetworkConfiguration: &mwaa.EnvironmentNetworkConfigurationArgs{
SecurityGroupIds: pulumi.StringArray{
exampleAwsSecurityGroup.Id,
},
SubnetIds: []pulumi.String(%!v(PANIC=Format method: fatal: A failure has occurred: unlowered splat expression @ example.pp:28,24-37)),
},
SourceBucketArn: pulumi.Any(exampleAwsS3Bucket.Arn),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.mwaa.Environment;
import com.pulumi.aws.mwaa.EnvironmentArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationDagProcessingLogsArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationSchedulerLogsArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationTaskLogsArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationWebserverLogsArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentLoggingConfigurationWorkerLogsArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentNetworkConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Environment("example", EnvironmentArgs.builder()
.dagS3Path("dags/")
.executionRoleArn(exampleAwsIamRole.arn())
.loggingConfiguration(EnvironmentLoggingConfigurationArgs.builder()
.dagProcessingLogs(EnvironmentLoggingConfigurationDagProcessingLogsArgs.builder()
.enabled(true)
.logLevel("DEBUG")
.build())
.schedulerLogs(EnvironmentLoggingConfigurationSchedulerLogsArgs.builder()
.enabled(true)
.logLevel("INFO")
.build())
.taskLogs(EnvironmentLoggingConfigurationTaskLogsArgs.builder()
.enabled(true)
.logLevel("WARNING")
.build())
.webserverLogs(EnvironmentLoggingConfigurationWebserverLogsArgs.builder()
.enabled(true)
.logLevel("ERROR")
.build())
.workerLogs(EnvironmentLoggingConfigurationWorkerLogsArgs.builder()
.enabled(true)
.logLevel("CRITICAL")
.build())
.build())
.name("example")
.networkConfiguration(EnvironmentNetworkConfigurationArgs.builder()
.securityGroupIds(exampleAwsSecurityGroup.id())
.subnetIds(private_.stream().map(element -> element.id()).collect(toList()))
.build())
.sourceBucketArn(exampleAwsS3Bucket.arn())
.build());
}
}
Example with tags
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.mwaa.Environment("example", {
dagS3Path: "dags/",
executionRoleArn: exampleAwsIamRole.arn,
name: "example",
networkConfiguration: {
securityGroupIds: [exampleAwsSecurityGroup.id],
subnetIds: _private.map(__item => __item.id),
},
sourceBucketArn: exampleAwsS3Bucket.arn,
tags: {
Name: "example",
Environment: "production",
},
});
import pulumi
import pulumi_aws as aws
example = aws.mwaa.Environment("example",
dag_s3_path="dags/",
execution_role_arn=example_aws_iam_role["arn"],
name="example",
network_configuration={
"security_group_ids": [example_aws_security_group["id"]],
"subnet_ids": [__item["id"] for __item in private],
},
source_bucket_arn=example_aws_s3_bucket["arn"],
tags={
"Name": "example",
"Environment": "production",
})
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Mwaa.Environment("example", new()
{
DagS3Path = "dags/",
ExecutionRoleArn = exampleAwsIamRole.Arn,
Name = "example",
NetworkConfiguration = new Aws.Mwaa.Inputs.EnvironmentNetworkConfigurationArgs
{
SecurityGroupIds = new[]
{
exampleAwsSecurityGroup.Id,
},
SubnetIds = @private.Select(__item => __item.Id).ToList(),
},
SourceBucketArn = exampleAwsS3Bucket.Arn,
Tags =
{
{ "Name", "example" },
{ "Environment", "production" },
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/mwaa"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := mwaa.NewEnvironment(ctx, "example", &mwaa.EnvironmentArgs{
DagS3Path: pulumi.String("dags/"),
ExecutionRoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Name: pulumi.String("example"),
NetworkConfiguration: &mwaa.EnvironmentNetworkConfigurationArgs{
SecurityGroupIds: pulumi.StringArray{
exampleAwsSecurityGroup.Id,
},
SubnetIds: []pulumi.String(%!v(PANIC=Format method: fatal: A failure has occurred: unlowered splat expression @ example.pp:6,24-37)),
},
SourceBucketArn: pulumi.Any(exampleAwsS3Bucket.Arn),
Tags: pulumi.StringMap{
"Name": pulumi.String("example"),
"Environment": pulumi.String("production"),
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.mwaa.Environment;
import com.pulumi.aws.mwaa.EnvironmentArgs;
import com.pulumi.aws.mwaa.inputs.EnvironmentNetworkConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Environment("example", EnvironmentArgs.builder()
.dagS3Path("dags/")
.executionRoleArn(exampleAwsIamRole.arn())
.name("example")
.networkConfiguration(EnvironmentNetworkConfigurationArgs.builder()
.securityGroupIds(exampleAwsSecurityGroup.id())
.subnetIds(private_.stream().map(element -> element.id()).collect(toList()))
.build())
.sourceBucketArn(exampleAwsS3Bucket.arn())
.tags(Map.ofEntries(
Map.entry("Name", "example"),
Map.entry("Environment", "production")
))
.build());
}
}
Import
Using pulumi import
, import MWAA Environment using Name
. For example:
$ pulumi import aws:mwaa/environment:Environment example MyAirflowEnvironment
Constructors
Properties
The airflow_configuration_options
parameter specifies airflow override options. Check the Official documentation for all possible configuration options.
Airflow version of your environment, will be set by default to the latest version that MWAA supports.
The relative path to the DAG folder on your Amazon S3 storage bucket. For example, dags. For more information, see Importing DAGs on Amazon MWAA.
Defines whether the VPC endpoints configured for the environment are created and managed by the customer or by AWS. If set to SERVICE
, Amazon MWAA will create and manage the required VPC endpoints in your VPC. If set to CUSTOMER
, you must create, and manage, the VPC endpoints for your VPC. Defaults to SERVICE
if not set.
Environment class for the cluster. Possible options are mw1.micro
, mw1.small
, mw1.medium
, mw1.large
. Will be set by default to mw1.small
. Please check the AWS Pricing for more information about the environment classes.
The Amazon Resource Name (ARN) of the task execution role that the Amazon MWAA and its environment can assume. Check the official AWS documentation for the detailed role specification.
The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption. Will be set to the ARN of the managed KMS key aws/airflow
by default. Please check the Official Documentation for more information.
The Apache Airflow logs you want to send to Amazon CloudWatch Logs. See logging_configuration
Block for details.
The maximum number of web servers that you want to run in your environment. Value need to be between 2
and 5
if environment_class
is not mw1.micro
, 1
otherwise.
The maximum number of workers that can be automatically scaled up. Value need to be between 1
and 25
. Will be 10
by default.
The minimum number of web servers that you want to run in your environment. Value need to be between 2
and 5
if environment_class
is not mw1.micro
, 1
otherwise.
The minimum number of workers that you want to run in your environment. Will be 1
by default.
Specifies the network configuration for your Apache Airflow Environment. This includes two private subnets as well as security groups for the Airflow environment. Each subnet requires internet connection, otherwise the deployment will fail. See network_configuration
Block for details.
The plugins.zip file version you want to use.
The relative path to the plugins.zip file on your Amazon S3 storage bucket. For example, plugins.zip. If a relative path is provided in the request, then plugins_s3_object_version is required. For more information, see Importing DAGs on Amazon MWAA.
The requirements.txt file version you want to use.
The relative path to the requirements.txt file on your Amazon S3 storage bucket. For example, requirements.txt. If a relative path is provided in the request, then requirements_s3_object_version is required. For more information, see Importing DAGs on Amazon MWAA.
The number of schedulers that you want to run in your environment. v2.0.2 and above accepts 2
- 5
, default 2
. v1.10.12 accepts 1
.
The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname.
The version of the startup shell script you want to use. You must specify the version ID that Amazon S3 assigns to the file every time you update the script.
The relative path to the script hosted in your bucket. The script runs as your environment starts before starting the Apache Airflow process. Use this script to install dependencies, modify configuration options, and set environment variables. See Using a startup script. Supported for environment versions 2.x and later.
Specifies whether the webserver should be accessible over the internet or via your specified VPC. Possible options: PRIVATE_ONLY
(default) and PUBLIC_ONLY
.
Specifies the start date for the weekly maintenance window.