pulumi-azure-kotlin/com.pulumi.azure.datafactory.kotlin/DataFlowArgs

DataFlowArgs

data class DataFlowArgs(val annotations: Output<List<String>>? = null, val dataFactoryId: Output<String>? = null, val description: Output<String>? = null, val folder: Output<String>? = null, val name: Output<String>? = null, val script: Output<String>? = null, val scriptLines: Output<List<String>>? = null, val sinks: Output<List<DataFlowSinkArgs>>? = null, val sources: Output<List<DataFlowSourceArgs>>? = null, val transformations: Output<List<DataFlowTransformationArgs>>? = null) : ConvertibleToJava<DataFlowArgs>

Manages a Data Flow inside an Azure Data Factory.

Example Usage

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azure.core.ResourceGroup;
import com.pulumi.azure.core.ResourceGroupArgs;
import com.pulumi.azure.storage.Account;
import com.pulumi.azure.storage.AccountArgs;
import com.pulumi.azure.datafactory.Factory;
import com.pulumi.azure.datafactory.FactoryArgs;
import com.pulumi.azure.datafactory.LinkedCustomService;
import com.pulumi.azure.datafactory.LinkedCustomServiceArgs;
import com.pulumi.azure.datafactory.DatasetJson;
import com.pulumi.azure.datafactory.DatasetJsonArgs;
import com.pulumi.azure.datafactory.inputs.DatasetJsonAzureBlobStorageLocationArgs;
import com.pulumi.azure.datafactory.FlowletDataFlow;
import com.pulumi.azure.datafactory.FlowletDataFlowArgs;
import com.pulumi.azure.datafactory.inputs.FlowletDataFlowSourceArgs;
import com.pulumi.azure.datafactory.inputs.FlowletDataFlowSourceLinkedServiceArgs;
import com.pulumi.azure.datafactory.inputs.FlowletDataFlowSinkArgs;
import com.pulumi.azure.datafactory.inputs.FlowletDataFlowSinkLinkedServiceArgs;
import com.pulumi.azure.datafactory.DataFlow;
import com.pulumi.azure.datafactory.DataFlowArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSourceArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSourceFlowletArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSourceDatasetArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSinkArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSinkFlowletArgs;
import com.pulumi.azure.datafactory.inputs.DataFlowSinkDatasetArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var exampleResourceGroup = new ResourceGroup("exampleResourceGroup", ResourceGroupArgs.builder()
            .location("West Europe")
            .build());
        var exampleAccount = new Account("exampleAccount", AccountArgs.builder()
            .location(exampleResourceGroup.location())
            .resourceGroupName(exampleResourceGroup.name())
            .accountTier("Standard")
            .accountReplicationType("LRS")
            .build());
        var exampleFactory = new Factory("exampleFactory", FactoryArgs.builder()
            .location(exampleResourceGroup.location())
            .resourceGroupName(exampleResourceGroup.name())
            .build());
        var exampleLinkedCustomService = new LinkedCustomService("exampleLinkedCustomService", LinkedCustomServiceArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .type("AzureBlobStorage")
            .typePropertiesJson(exampleAccount.primaryConnectionString().applyValue(primaryConnectionString -> """
{
  "connectionString": "%s"
}
", primaryConnectionString)))
            .build());
        var example1DatasetJson = new DatasetJson("example1DatasetJson", DatasetJsonArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .linkedServiceName(exampleLinkedCustomService.name())
            .azureBlobStorageLocation(DatasetJsonAzureBlobStorageLocationArgs.builder()
                .container("container")
                .path("foo/bar/")
                .filename("foo.txt")
                .build())
            .encoding("UTF-8")
            .build());
        var example2DatasetJson = new DatasetJson("example2DatasetJson", DatasetJsonArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .linkedServiceName(exampleLinkedCustomService.name())
            .azureBlobStorageLocation(DatasetJsonAzureBlobStorageLocationArgs.builder()
                .container("container")
                .path("foo/bar/")
                .filename("bar.txt")
                .build())
            .encoding("UTF-8")
            .build());
        var example1FlowletDataFlow = new FlowletDataFlow("example1FlowletDataFlow", FlowletDataFlowArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .sources(FlowletDataFlowSourceArgs.builder()
                .name("source1")
                .linkedService(FlowletDataFlowSourceLinkedServiceArgs.builder()
                    .name(exampleLinkedCustomService.name())
                    .build())
                .build())
            .sinks(FlowletDataFlowSinkArgs.builder()
                .name("sink1")
                .linkedService(FlowletDataFlowSinkLinkedServiceArgs.builder()
                    .name(exampleLinkedCustomService.name())
                    .build())
                .build())
            .script("""
source(
  allowSchemaDrift: true,
  validateSchema: false,
  limit: 100,
  ignoreNoFilesFound: false,
  documentForm: 'documentPerLine') ~> source1
source1 sink(
  allowSchemaDrift: true,
  validateSchema: false,
  skipDuplicateMapInputs: true,
  skipDuplicateMapOutputs: true) ~> sink1
            """)
            .build());
        var example2FlowletDataFlow = new FlowletDataFlow("example2FlowletDataFlow", FlowletDataFlowArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .sources(FlowletDataFlowSourceArgs.builder()
                .name("source1")
                .linkedService(FlowletDataFlowSourceLinkedServiceArgs.builder()
                    .name(exampleLinkedCustomService.name())
                    .build())
                .build())
            .sinks(FlowletDataFlowSinkArgs.builder()
                .name("sink1")
                .linkedService(FlowletDataFlowSinkLinkedServiceArgs.builder()
                    .name(exampleLinkedCustomService.name())
                    .build())
                .build())
            .script("""
source(
  allowSchemaDrift: true,
  validateSchema: false,
  limit: 100,
  ignoreNoFilesFound: false,
  documentForm: 'documentPerLine') ~> source1
source1 sink(
  allowSchemaDrift: true,
  validateSchema: false,
  skipDuplicateMapInputs: true,
  skipDuplicateMapOutputs: true) ~> sink1
            """)
            .build());
        var exampleDataFlow = new DataFlow("exampleDataFlow", DataFlowArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .sources(DataFlowSourceArgs.builder()
                .name("source1")
                .flowlet(DataFlowSourceFlowletArgs.builder()
                    .name(example1FlowletDataFlow.name())
                    .parameters(Map.of("Key1", "value1"))
                    .build())
                .dataset(DataFlowSourceDatasetArgs.builder()
                    .name(example1DatasetJson.name())
                    .build())
                .build())
            .sinks(DataFlowSinkArgs.builder()
                .name("sink1")
                .flowlet(DataFlowSinkFlowletArgs.builder()
                    .name(example2FlowletDataFlow.name())
                    .parameters(Map.of("Key1", "value1"))
                    .build())
                .dataset(DataFlowSinkDatasetArgs.builder()
                    .name(example2DatasetJson.name())
                    .build())
                .build())
            .script("""
source(
  allowSchemaDrift: true,
  validateSchema: false,
  limit: 100,
  ignoreNoFilesFound: false,
  documentForm: 'documentPerLine') ~> source1
source1 sink(
  allowSchemaDrift: true,
  validateSchema: false,
  skipDuplicateMapInputs: true,
  skipDuplicateMapOutputs: true) ~> sink1
            """)
            .build());
    }
}
Content copied to clipboard

Import

Data Factory Data Flow can be imported using the resource id, e.g.

$ pulumi import azure:datafactory/dataFlow:DataFlow example /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/example/providers/Microsoft.DataFactory/factories/example/dataflows/example
Content copied to clipboard

Constructors

Link copied to clipboard
fun DataFlowArgs(annotations: Output<List<String>>? = null, dataFactoryId: Output<String>? = null, description: Output<String>? = null, folder: Output<String>? = null, name: Output<String>? = null, script: Output<String>? = null, scriptLines: Output<List<String>>? = null, sinks: Output<List<DataFlowSinkArgs>>? = null, sources: Output<List<DataFlowSourceArgs>>? = null, transformations: Output<List<DataFlowTransformationArgs>>? = null)

Functions

Link copied to clipboard
open override fun toJava(): DataFlowArgs

Properties

Link copied to clipboard
val annotations: Output<List<String>>? = null

List of tags that can be used for describing the Data Factory Data Flow.

Link copied to clipboard
val dataFactoryId: Output<String>? = null

The ID of Data Factory in which to associate the Data Flow with. Changing this forces a new resource.

Link copied to clipboard
val description: Output<String>? = null

The description for the Data Factory Data Flow.

Link copied to clipboard
val folder: Output<String>? = null

The folder that this Data Flow is in. If not specified, the Data Flow will appear at the root level.

Link copied to clipboard
val name: Output<String>? = null

Specifies the name of the Data Factory Data Flow. Changing this forces a new resource to be created.

Link copied to clipboard
val script: Output<String>? = null

The script for the Data Factory Data Flow.

Link copied to clipboard
val scriptLines: Output<List<String>>? = null

The script lines for the Data Factory Data Flow.

Link copied to clipboard
val sinks: Output<List<DataFlowSinkArgs>>? = null

One or more sink blocks as defined below.

Link copied to clipboard
val sources: Output<List<DataFlowSourceArgs>>? = null

One or more source blocks as defined below.

Link copied to clipboard
val transformations: Output<List<DataFlowTransformationArgs>>? = null

One or more transformation blocks as defined below.