pulumi-aws-kotlin/com.pulumi.aws.glue.kotlin/Crawler

Crawler

class Crawler : KotlinCustomResource

Manages a Glue Crawler. More information can be found in the AWS Glue Developer Guide

Example Usage

DynamoDB Target Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerDynamodbTargetArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Crawler("example", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.example().name())
            .role(aws_iam_role.example().arn())
            .dynamodbTargets(CrawlerDynamodbTargetArgs.builder()
                .path("table-name")
                .build())
            .build());
    }
}
Content copied to clipboard

JDBC Target Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerJdbcTargetArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Crawler("example", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.example().name())
            .role(aws_iam_role.example().arn())
            .jdbcTargets(CrawlerJdbcTargetArgs.builder()
                .connectionName(aws_glue_connection.example().name())
                .path("database-name/%")
                .build())
            .build());
    }
}
Content copied to clipboard

S3 Target Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Crawler("example", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.example().name())
            .role(aws_iam_role.example().arn())
            .s3Targets(CrawlerS3TargetArgs.builder()
                .path(String.format("s3://%s", aws_s3_bucket.example().bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard

Catalog Target Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerCatalogTargetArgs;
import com.pulumi.aws.glue.inputs.CrawlerSchemaChangePolicyArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Crawler("example", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.example().name())
            .role(aws_iam_role.example().arn())
            .catalogTargets(CrawlerCatalogTargetArgs.builder()
                .databaseName(aws_glue_catalog_database.example().name())
                .tables(aws_glue_catalog_table.example().name())
                .build())
            .schemaChangePolicy(CrawlerSchemaChangePolicyArgs.builder()
                .deleteBehavior("LOG")
                .build())
            .configuration("""
{
  "Version":1.0,
  "Grouping": {
    "TableGroupingPolicy": "CombineCompatibleSchemas"
  }
}
            """)
            .build());
    }
}
Content copied to clipboard

MongoDB Target Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerMongodbTargetArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var example = new Crawler("example", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.example().name())
            .role(aws_iam_role.example().arn())
            .mongodbTargets(CrawlerMongodbTargetArgs.builder()
                .connectionName(aws_glue_connection.example().name())
                .path("database-name/%")
                .build())
            .build());
    }
}
Content copied to clipboard

Configuration Settings Example

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var eventsCrawler = new Crawler("eventsCrawler", CrawlerArgs.builder()
            .databaseName(aws_glue_catalog_database.glue_database().name())
            .schedule("cron(0 1 * * ? *)")
            .role(aws_iam_role.glue_role().arn())
            .tags(var_.tags())
            .configuration(serializeJson(
                jsonObject(
                    jsonProperty("Grouping", jsonObject(
                        jsonProperty("TableGroupingPolicy", "CombineCompatibleSchemas")
                    )),
                    jsonProperty("CrawlerOutput", jsonObject(
                        jsonProperty("Partitions", jsonObject(
                            jsonProperty("AddOrUpdateBehavior", "InheritFromTable")
                        ))
                    )),
                    jsonProperty("Version", 1)
                )))
            .s3Targets(CrawlerS3TargetArgs.builder()
                .path(String.format("s3://%s", aws_s3_bucket.data_lake_bucket().bucket()))
                .build())
            .build());
    }
}
Content copied to clipboard

Import

Using pulumi import, import Glue Crawlers using name. For example:

$ pulumi import aws:glue/crawler:Crawler MyJob MyJob
Content copied to clipboard

Properties

Link copied to clipboard
val arn: Output<String>

The ARN of the crawler

Link copied to clipboard
Link copied to clipboard
val classifiers: Output<List<String>>?

List of custom classifiers. By default, all AWS classifiers are included in a crawl, but these custom classifiers always override the default classifiers for a given classification.

Link copied to clipboard
val configuration: Output<String>?

JSON string of configuration information. For more details see Setting Crawler Configuration Options.

Link copied to clipboard
val databaseName: Output<String>

Glue database where results are written.

Link copied to clipboard
val deltaTargets: Output<List<CrawlerDeltaTarget>>?

List of nested Delta Lake target arguments. See Delta Target below.

Link copied to clipboard
val description: Output<String>?

Description of the crawler.

Link copied to clipboard
val dynamodbTargets: Output<List<CrawlerDynamodbTarget>>?

List of nested DynamoDB target arguments. See Dynamodb Target below.

Link copied to clipboard
val hudiTargets: Output<List<CrawlerHudiTarget>>?

List nested Hudi target arguments. See Iceberg Target below.

Link copied to clipboard
val icebergTargets: Output<List<CrawlerIcebergTarget>>?

List nested Iceberg target arguments. See Iceberg Target below.

Link copied to clipboard
val id: Output<String>
Link copied to clipboard
val jdbcTargets: Output<List<CrawlerJdbcTarget>>?

List of nested JBDC target arguments. See JDBC Target below.

Link copied to clipboard
val lakeFormationConfiguration: Output<CrawlerLakeFormationConfiguration>?

Specifies Lake Formation configuration settings for the crawler. See Lake Formation Configuration below.

Link copied to clipboard
val lineageConfiguration: Output<CrawlerLineageConfiguration>?

Specifies data lineage configuration settings for the crawler. See Lineage Configuration below.

Link copied to clipboard
val mongodbTargets: Output<List<CrawlerMongodbTarget>>?

List nested MongoDB target arguments. See MongoDB Target below.

Link copied to clipboard
val name: Output<String>

Name of the crawler.

Link copied to clipboard
val pulumiChildResources: Set<KotlinResource>
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
val recrawlPolicy: Output<CrawlerRecrawlPolicy>?

A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.. See Recrawl Policy below.

Link copied to clipboard
val role: Output<String>

The IAM role friendly name (including path without leading slash), or ARN of an IAM role, used by the crawler to access other resources.

Link copied to clipboard
val s3Targets: Output<List<CrawlerS3Target>>?

List nested Amazon S3 target arguments. See S3 Target below.

Link copied to clipboard
val schedule: Output<String>?

A cron expression used to specify the schedule. For more information, see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).

Link copied to clipboard
val schemaChangePolicy: Output<CrawlerSchemaChangePolicy>?

Policy for the crawler's update and deletion behavior. See Schema Change Policy below.

Link copied to clipboard
val securityConfiguration: Output<String>?

The name of Security Configuration to be used by the crawler

Link copied to clipboard
val tablePrefix: Output<String>?

The table prefix used for catalog tables that are created.

Link copied to clipboard
val tags: Output<Map<String, String>>?

Key-value map of resource tags. .If configured with a provider default_tags configuration block present, tags with matching keys will overwrite those defined at the provider-level.

Link copied to clipboard
val tagsAll: Output<Map<String, String>>

A map of tags assigned to the resource, including those inherited from the provider default_tags configuration block.

Link copied to clipboard
val urn: Output<String>