pulumi-azure-kotlin/com.pulumi.azure.datafactory.kotlin/LinkedServiceAzureDatabricks

LinkedServiceAzureDatabricks

class LinkedServiceAzureDatabricks : KotlinCustomResource

Manages a Linked Service (connection) between Azure Databricks and Azure Data Factory.

Example Usage

With Managed Identity & New Cluster

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azure.core.ResourceGroup;
import com.pulumi.azure.core.ResourceGroupArgs;
import com.pulumi.azure.datafactory.Factory;
import com.pulumi.azure.datafactory.FactoryArgs;
import com.pulumi.azure.datafactory.inputs.FactoryIdentityArgs;
import com.pulumi.azure.databricks.Workspace;
import com.pulumi.azure.databricks.WorkspaceArgs;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricks;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricksArgs;
import com.pulumi.azure.datafactory.inputs.LinkedServiceAzureDatabricksNewClusterConfigArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var exampleResourceGroup = new ResourceGroup("exampleResourceGroup", ResourceGroupArgs.builder()
            .location("East US")
            .build());
        var exampleFactory = new Factory("exampleFactory", FactoryArgs.builder()
            .location(exampleResourceGroup.location())
            .resourceGroupName(exampleResourceGroup.name())
            .identity(FactoryIdentityArgs.builder()
                .type("SystemAssigned")
                .build())
            .build());
        var exampleWorkspace = new Workspace("exampleWorkspace", WorkspaceArgs.builder()
            .resourceGroupName(exampleResourceGroup.name())
            .location(exampleResourceGroup.location())
            .sku("standard")
            .build());
        var msiLinked = new LinkedServiceAzureDatabricks("msiLinked", LinkedServiceAzureDatabricksArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .description("ADB Linked Service via MSI")
            .adbDomain(exampleWorkspace.workspaceUrl().applyValue(workspaceUrl -> String.format("https://%s", workspaceUrl)))
            .msiWorkSpaceResourceId(exampleWorkspace.id())
            .newClusterConfig(LinkedServiceAzureDatabricksNewClusterConfigArgs.builder()
                .nodeType("Standard_NC12")
                .clusterVersion("5.5.x-gpu-scala2.11")
                .minNumberOfWorkers(1)
                .maxNumberOfWorkers(5)
                .driverNodeType("Standard_NC12")
                .logDestination("dbfs:/logs")
                .customTags(Map.ofEntries(
                    Map.entry("custom_tag1", "sct_value_1"),
                    Map.entry("custom_tag2", "sct_value_2")
                ))
                .sparkConfig(Map.ofEntries(
                    Map.entry("config1", "value1"),
                    Map.entry("config2", "value2")
                ))
                .sparkEnvironmentVariables(Map.ofEntries(
                    Map.entry("envVar1", "value1"),
                    Map.entry("envVar2", "value2")
                ))
                .initScripts(
                    "init.sh",
                    "init2.sh")
                .build())
            .build());
    }
}
Content copied to clipboard

With Access Token & Existing Cluster

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azure.core.ResourceGroup;
import com.pulumi.azure.core.ResourceGroupArgs;
import com.pulumi.azure.datafactory.Factory;
import com.pulumi.azure.datafactory.FactoryArgs;
import com.pulumi.azure.databricks.Workspace;
import com.pulumi.azure.databricks.WorkspaceArgs;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricks;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricksArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var exampleResourceGroup = new ResourceGroup("exampleResourceGroup", ResourceGroupArgs.builder()
            .location("East US")
            .build());
        var exampleFactory = new Factory("exampleFactory", FactoryArgs.builder()
            .location(exampleResourceGroup.location())
            .resourceGroupName(exampleResourceGroup.name())
            .build());
        var exampleWorkspace = new Workspace("exampleWorkspace", WorkspaceArgs.builder()
            .resourceGroupName(exampleResourceGroup.name())
            .location(exampleResourceGroup.location())
            .sku("standard")
            .build());
        var atLinked = new LinkedServiceAzureDatabricks("atLinked", LinkedServiceAzureDatabricksArgs.builder()
            .dataFactoryId(exampleFactory.id())
            .description("ADB Linked Service via Access Token")
            .existingClusterId("0308-201146-sly615")
            .accessToken("SomeDatabricksAccessToken")
            .adbDomain(exampleWorkspace.workspaceUrl().applyValue(workspaceUrl -> String.format("https://%s", workspaceUrl)))
            .build());
    }
}
Content copied to clipboard

Import

Data Factory Linked Services can be imported using the resource id, e.g.

$ pulumi import azure:datafactory/linkedServiceAzureDatabricks:LinkedServiceAzureDatabricks example /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/example/providers/Microsoft.DataFactory/factories/example/linkedservices/example
Content copied to clipboard

Properties

Link copied to clipboard
val accessToken: Output<String>?

Authenticate to ADB via an access token.

Link copied to clipboard
val adbDomain: Output<String>

The domain URL of the databricks instance.

Link copied to clipboard
val additionalProperties: Output<Map<String, String>>?

A map of additional properties to associate with the Data Factory Linked Service.

Link copied to clipboard
val annotations: Output<List<String>>?

List of tags that can be used for describing the Data Factory Linked Service.

Link copied to clipboard
val dataFactoryId: Output<String>

The Data Factory ID in which to associate the Linked Service with. Changing this forces a new resource.

Link copied to clipboard
val description: Output<String>?

The description for the Data Factory Linked Service.

Link copied to clipboard
val existingClusterId: Output<String>?

The cluster_id of an existing cluster within the linked ADB instance.

Link copied to clipboard
val id: Output<String>
Link copied to clipboard
val instancePool: Output<LinkedServiceAzureDatabricksInstancePool>?

Leverages an instance pool within the linked ADB instance as defined by instance_pool block below.

Link copied to clipboard
val integrationRuntimeName: Output<String>?

The integration runtime reference to associate with the Data Factory Linked Service.

Link copied to clipboard
val keyVaultPassword: Output<LinkedServiceAzureDatabricksKeyVaultPassword>?

Authenticate to ADB via Azure Key Vault Linked Service as defined in the key_vault_password block below.

Link copied to clipboard
val msiWorkSpaceResourceId: Output<String>?

Authenticate to ADB via managed service identity.

Link copied to clipboard
val name: Output<String>

Specifies the name of the Data Factory Linked Service. Changing this forces a new resource to be created. Must be unique within a data factory. See the Microsoft documentation for all restrictions.

Link copied to clipboard
val newClusterConfig: Output<LinkedServiceAzureDatabricksNewClusterConfig>?

Creates new clusters within the linked ADB instance as defined in the new_cluster_config block below.

Link copied to clipboard
val parameters: Output<Map<String, String>>?

A map of parameters to associate with the Data Factory Linked Service.

Link copied to clipboard
val pulumiChildResources: Set<KotlinResource>
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
val urn: Output<String>