Linked
data class LinkedServiceAzureDatabricksArgs(val accessToken: Output<String>? = null, val adbDomain: Output<String>? = null, val additionalProperties: Output<Map<String, String>>? = null, val annotations: Output<List<String>>? = null, val dataFactoryId: Output<String>? = null, val description: Output<String>? = null, val existingClusterId: Output<String>? = null, val instancePool: Output<LinkedServiceAzureDatabricksInstancePoolArgs>? = null, val integrationRuntimeName: Output<String>? = null, val keyVaultPassword: Output<LinkedServiceAzureDatabricksKeyVaultPasswordArgs>? = null, val msiWorkSpaceResourceId: Output<String>? = null, val name: Output<String>? = null, val newClusterConfig: Output<LinkedServiceAzureDatabricksNewClusterConfigArgs>? = null, val parameters: Output<Map<String, String>>? = null) : ConvertibleToJava<LinkedServiceAzureDatabricksArgs>
Manages a Linked Service (connection) between Azure Databricks and Azure Data Factory.
Example Usage
With Managed Identity & New Cluster
import * as pulumi from "@pulumi/pulumi";
import * as azure from "@pulumi/azure";
const example = new azure.core.ResourceGroup("example", {
name: "example",
location: "East US",
});
//Create a Linked Service using managed identity and new cluster config
const exampleFactory = new azure.datafactory.Factory("example", {
name: "TestDtaFactory92783401247",
location: example.location,
resourceGroupName: example.name,
identity: {
type: "SystemAssigned",
},
});
//Create a databricks instance
const exampleWorkspace = new azure.databricks.Workspace("example", {
name: "databricks-test",
resourceGroupName: example.name,
location: example.location,
sku: "standard",
});
const msiLinked = new azure.datafactory.LinkedServiceAzureDatabricks("msi_linked", {
name: "ADBLinkedServiceViaMSI",
dataFactoryId: exampleFactory.id,
description: "ADB Linked Service via MSI",
adbDomain: pulumi.interpolate`https://${exampleWorkspace.workspaceUrl}`,
msiWorkSpaceResourceId: exampleWorkspace.id,
newClusterConfig: {
nodeType: "Standard_NC12",
clusterVersion: "5.5.x-gpu-scala2.11",
minNumberOfWorkers: 1,
maxNumberOfWorkers: 5,
driverNodeType: "Standard_NC12",
logDestination: "dbfs:/logs",
customTags: {
custom_tag1: "sct_value_1",
custom_tag2: "sct_value_2",
},
sparkConfig: {
config1: "value1",
config2: "value2",
},
sparkEnvironmentVariables: {
envVar1: "value1",
envVar2: "value2",
},
initScripts: [
"init.sh",
"init2.sh",
],
},
});Content copied to clipboard
import pulumi
import pulumi_azure as azure
example = azure.core.ResourceGroup("example",
name="example",
location="East US")
#Create a Linked Service using managed identity and new cluster config
example_factory = azure.datafactory.Factory("example",
name="TestDtaFactory92783401247",
location=example.location,
resource_group_name=example.name,
identity={
"type": "SystemAssigned",
})
#Create a databricks instance
example_workspace = azure.databricks.Workspace("example",
name="databricks-test",
resource_group_name=example.name,
location=example.location,
sku="standard")
msi_linked = azure.datafactory.LinkedServiceAzureDatabricks("msi_linked",
name="ADBLinkedServiceViaMSI",
data_factory_id=example_factory.id,
description="ADB Linked Service via MSI",
adb_domain=example_workspace.workspace_url.apply(lambda workspace_url: f"https://{workspace_url}"),
msi_work_space_resource_id=example_workspace.id,
new_cluster_config={
"node_type": "Standard_NC12",
"cluster_version": "5.5.x-gpu-scala2.11",
"min_number_of_workers": 1,
"max_number_of_workers": 5,
"driver_node_type": "Standard_NC12",
"log_destination": "dbfs:/logs",
"custom_tags": {
"custom_tag1": "sct_value_1",
"custom_tag2": "sct_value_2",
},
"spark_config": {
"config1": "value1",
"config2": "value2",
},
"spark_environment_variables": {
"envVar1": "value1",
"envVar2": "value2",
},
"init_scripts": [
"init.sh",
"init2.sh",
],
})Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Azure = Pulumi.Azure;
return await Deployment.RunAsync(() =>
{
var example = new Azure.Core.ResourceGroup("example", new()
{
Name = "example",
Location = "East US",
});
//Create a Linked Service using managed identity and new cluster config
var exampleFactory = new Azure.DataFactory.Factory("example", new()
{
Name = "TestDtaFactory92783401247",
Location = example.Location,
ResourceGroupName = example.Name,
Identity = new Azure.DataFactory.Inputs.FactoryIdentityArgs
{
Type = "SystemAssigned",
},
});
//Create a databricks instance
var exampleWorkspace = new Azure.DataBricks.Workspace("example", new()
{
Name = "databricks-test",
ResourceGroupName = example.Name,
Location = example.Location,
Sku = "standard",
});
var msiLinked = new Azure.DataFactory.LinkedServiceAzureDatabricks("msi_linked", new()
{
Name = "ADBLinkedServiceViaMSI",
DataFactoryId = exampleFactory.Id,
Description = "ADB Linked Service via MSI",
AdbDomain = exampleWorkspace.WorkspaceUrl.Apply(workspaceUrl => $"https://{workspaceUrl}"),
MsiWorkSpaceResourceId = exampleWorkspace.Id,
NewClusterConfig = new Azure.DataFactory.Inputs.LinkedServiceAzureDatabricksNewClusterConfigArgs
{
NodeType = "Standard_NC12",
ClusterVersion = "5.5.x-gpu-scala2.11",
MinNumberOfWorkers = 1,
MaxNumberOfWorkers = 5,
DriverNodeType = "Standard_NC12",
LogDestination = "dbfs:/logs",
CustomTags =
{
{ "custom_tag1", "sct_value_1" },
{ "custom_tag2", "sct_value_2" },
},
SparkConfig =
{
{ "config1", "value1" },
{ "config2", "value2" },
},
SparkEnvironmentVariables =
{
{ "envVar1", "value1" },
{ "envVar2", "value2" },
},
InitScripts = new[]
{
"init.sh",
"init2.sh",
},
},
});
});Content copied to clipboard
package main
import (
"fmt"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/core"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/databricks"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/datafactory"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := core.NewResourceGroup(ctx, "example", &core.ResourceGroupArgs{
Name: pulumi.String("example"),
Location: pulumi.String("East US"),
})
if err != nil {
return err
}
// Create a Linked Service using managed identity and new cluster config
exampleFactory, err := datafactory.NewFactory(ctx, "example", &datafactory.FactoryArgs{
Name: pulumi.String("TestDtaFactory92783401247"),
Location: example.Location,
ResourceGroupName: example.Name,
Identity: &datafactory.FactoryIdentityArgs{
Type: pulumi.String("SystemAssigned"),
},
})
if err != nil {
return err
}
// Create a databricks instance
exampleWorkspace, err := databricks.NewWorkspace(ctx, "example", &databricks.WorkspaceArgs{
Name: pulumi.String("databricks-test"),
ResourceGroupName: example.Name,
Location: example.Location,
Sku: pulumi.String("standard"),
})
if err != nil {
return err
}
_, err = datafactory.NewLinkedServiceAzureDatabricks(ctx, "msi_linked", &datafactory.LinkedServiceAzureDatabricksArgs{
Name: pulumi.String("ADBLinkedServiceViaMSI"),
DataFactoryId: exampleFactory.ID(),
Description: pulumi.String("ADB Linked Service via MSI"),
AdbDomain: exampleWorkspace.WorkspaceUrl.ApplyT(func(workspaceUrl string) (string, error) {
return fmt.Sprintf("https://%v", workspaceUrl), nil
}).(pulumi.StringOutput),
MsiWorkSpaceResourceId: exampleWorkspace.ID(),
NewClusterConfig: &datafactory.LinkedServiceAzureDatabricksNewClusterConfigArgs{
NodeType: pulumi.String("Standard_NC12"),
ClusterVersion: pulumi.String("5.5.x-gpu-scala2.11"),
MinNumberOfWorkers: pulumi.Int(1),
MaxNumberOfWorkers: pulumi.Int(5),
DriverNodeType: pulumi.String("Standard_NC12"),
LogDestination: pulumi.String("dbfs:/logs"),
CustomTags: pulumi.StringMap{
"custom_tag1": pulumi.String("sct_value_1"),
"custom_tag2": pulumi.String("sct_value_2"),
},
SparkConfig: pulumi.StringMap{
"config1": pulumi.String("value1"),
"config2": pulumi.String("value2"),
},
SparkEnvironmentVariables: pulumi.StringMap{
"envVar1": pulumi.String("value1"),
"envVar2": pulumi.String("value2"),
},
InitScripts: pulumi.StringArray{
pulumi.String("init.sh"),
pulumi.String("init2.sh"),
},
},
})
if err != nil {
return err
}
return nil
})
}Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azure.core.ResourceGroup;
import com.pulumi.azure.core.ResourceGroupArgs;
import com.pulumi.azure.datafactory.Factory;
import com.pulumi.azure.datafactory.FactoryArgs;
import com.pulumi.azure.datafactory.inputs.FactoryIdentityArgs;
import com.pulumi.azure.databricks.Workspace;
import com.pulumi.azure.databricks.WorkspaceArgs;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricks;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricksArgs;
import com.pulumi.azure.datafactory.inputs.LinkedServiceAzureDatabricksNewClusterConfigArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new ResourceGroup("example", ResourceGroupArgs.builder()
.name("example")
.location("East US")
.build());
//Create a Linked Service using managed identity and new cluster config
var exampleFactory = new Factory("exampleFactory", FactoryArgs.builder()
.name("TestDtaFactory92783401247")
.location(example.location())
.resourceGroupName(example.name())
.identity(FactoryIdentityArgs.builder()
.type("SystemAssigned")
.build())
.build());
//Create a databricks instance
var exampleWorkspace = new Workspace("exampleWorkspace", WorkspaceArgs.builder()
.name("databricks-test")
.resourceGroupName(example.name())
.location(example.location())
.sku("standard")
.build());
var msiLinked = new LinkedServiceAzureDatabricks("msiLinked", LinkedServiceAzureDatabricksArgs.builder()
.name("ADBLinkedServiceViaMSI")
.dataFactoryId(exampleFactory.id())
.description("ADB Linked Service via MSI")
.adbDomain(exampleWorkspace.workspaceUrl().applyValue(_workspaceUrl -> String.format("https://%s", _workspaceUrl)))
.msiWorkSpaceResourceId(exampleWorkspace.id())
.newClusterConfig(LinkedServiceAzureDatabricksNewClusterConfigArgs.builder()
.nodeType("Standard_NC12")
.clusterVersion("5.5.x-gpu-scala2.11")
.minNumberOfWorkers(1)
.maxNumberOfWorkers(5)
.driverNodeType("Standard_NC12")
.logDestination("dbfs:/logs")
.customTags(Map.ofEntries(
Map.entry("custom_tag1", "sct_value_1"),
Map.entry("custom_tag2", "sct_value_2")
))
.sparkConfig(Map.ofEntries(
Map.entry("config1", "value1"),
Map.entry("config2", "value2")
))
.sparkEnvironmentVariables(Map.ofEntries(
Map.entry("envVar1", "value1"),
Map.entry("envVar2", "value2")
))
.initScripts(
"init.sh",
"init2.sh")
.build())
.build());
}
}Content copied to clipboard
resources:
example:
type: azure:core:ResourceGroup
properties:
name: example
location: East US
#Create a Linked Service using managed identity and new cluster config
exampleFactory:
type: azure:datafactory:Factory
name: example
properties:
name: TestDtaFactory92783401247
location: ${example.location}
resourceGroupName: ${example.name}
identity:
type: SystemAssigned
#Create a databricks instance
exampleWorkspace:
type: azure:databricks:Workspace
name: example
properties:
name: databricks-test
resourceGroupName: ${example.name}
location: ${example.location}
sku: standard
msiLinked:
type: azure:datafactory:LinkedServiceAzureDatabricks
name: msi_linked
properties:
name: ADBLinkedServiceViaMSI
dataFactoryId: ${exampleFactory.id}
description: ADB Linked Service via MSI
adbDomain: https://${exampleWorkspace.workspaceUrl}
msiWorkSpaceResourceId: ${exampleWorkspace.id}
newClusterConfig:
nodeType: Standard_NC12
clusterVersion: 5.5.x-gpu-scala2.11
minNumberOfWorkers: 1
maxNumberOfWorkers: 5
driverNodeType: Standard_NC12
logDestination: dbfs:/logs
customTags:
custom_tag1: sct_value_1
custom_tag2: sct_value_2
sparkConfig:
config1: value1
config2: value2
sparkEnvironmentVariables:
envVar1: value1
envVar2: value2
initScripts:
- init.sh
- init2.shContent copied to clipboard
With Access Token & Existing Cluster
import * as pulumi from "@pulumi/pulumi";
import * as azure from "@pulumi/azure";
const example = new azure.core.ResourceGroup("example", {
name: "example",
location: "East US",
});
//Link to an existing cluster via access token
const exampleFactory = new azure.datafactory.Factory("example", {
name: "TestDtaFactory92783401247",
location: example.location,
resourceGroupName: example.name,
});
//Create a databricks instance
const exampleWorkspace = new azure.databricks.Workspace("example", {
name: "databricks-test",
resourceGroupName: example.name,
location: example.location,
sku: "standard",
});
const atLinked = new azure.datafactory.LinkedServiceAzureDatabricks("at_linked", {
name: "ADBLinkedServiceViaAccessToken",
dataFactoryId: exampleFactory.id,
description: "ADB Linked Service via Access Token",
existingClusterId: "0308-201146-sly615",
accessToken: "SomeDatabricksAccessToken",
adbDomain: pulumi.interpolate`https://${exampleWorkspace.workspaceUrl}`,
});Content copied to clipboard
import pulumi
import pulumi_azure as azure
example = azure.core.ResourceGroup("example",
name="example",
location="East US")
#Link to an existing cluster via access token
example_factory = azure.datafactory.Factory("example",
name="TestDtaFactory92783401247",
location=example.location,
resource_group_name=example.name)
#Create a databricks instance
example_workspace = azure.databricks.Workspace("example",
name="databricks-test",
resource_group_name=example.name,
location=example.location,
sku="standard")
at_linked = azure.datafactory.LinkedServiceAzureDatabricks("at_linked",
name="ADBLinkedServiceViaAccessToken",
data_factory_id=example_factory.id,
description="ADB Linked Service via Access Token",
existing_cluster_id="0308-201146-sly615",
access_token="SomeDatabricksAccessToken",
adb_domain=example_workspace.workspace_url.apply(lambda workspace_url: f"https://{workspace_url}"))Content copied to clipboard
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Azure = Pulumi.Azure;
return await Deployment.RunAsync(() =>
{
var example = new Azure.Core.ResourceGroup("example", new()
{
Name = "example",
Location = "East US",
});
//Link to an existing cluster via access token
var exampleFactory = new Azure.DataFactory.Factory("example", new()
{
Name = "TestDtaFactory92783401247",
Location = example.Location,
ResourceGroupName = example.Name,
});
//Create a databricks instance
var exampleWorkspace = new Azure.DataBricks.Workspace("example", new()
{
Name = "databricks-test",
ResourceGroupName = example.Name,
Location = example.Location,
Sku = "standard",
});
var atLinked = new Azure.DataFactory.LinkedServiceAzureDatabricks("at_linked", new()
{
Name = "ADBLinkedServiceViaAccessToken",
DataFactoryId = exampleFactory.Id,
Description = "ADB Linked Service via Access Token",
ExistingClusterId = "0308-201146-sly615",
AccessToken = "SomeDatabricksAccessToken",
AdbDomain = exampleWorkspace.WorkspaceUrl.Apply(workspaceUrl => $"https://{workspaceUrl}"),
});
});Content copied to clipboard
package main
import (
"fmt"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/core"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/databricks"
"github.com/pulumi/pulumi-azure/sdk/v6/go/azure/datafactory"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := core.NewResourceGroup(ctx, "example", &core.ResourceGroupArgs{
Name: pulumi.String("example"),
Location: pulumi.String("East US"),
})
if err != nil {
return err
}
// Link to an existing cluster via access token
exampleFactory, err := datafactory.NewFactory(ctx, "example", &datafactory.FactoryArgs{
Name: pulumi.String("TestDtaFactory92783401247"),
Location: example.Location,
ResourceGroupName: example.Name,
})
if err != nil {
return err
}
// Create a databricks instance
exampleWorkspace, err := databricks.NewWorkspace(ctx, "example", &databricks.WorkspaceArgs{
Name: pulumi.String("databricks-test"),
ResourceGroupName: example.Name,
Location: example.Location,
Sku: pulumi.String("standard"),
})
if err != nil {
return err
}
_, err = datafactory.NewLinkedServiceAzureDatabricks(ctx, "at_linked", &datafactory.LinkedServiceAzureDatabricksArgs{
Name: pulumi.String("ADBLinkedServiceViaAccessToken"),
DataFactoryId: exampleFactory.ID(),
Description: pulumi.String("ADB Linked Service via Access Token"),
ExistingClusterId: pulumi.String("0308-201146-sly615"),
AccessToken: pulumi.String("SomeDatabricksAccessToken"),
AdbDomain: exampleWorkspace.WorkspaceUrl.ApplyT(func(workspaceUrl string) (string, error) {
return fmt.Sprintf("https://%v", workspaceUrl), nil
}).(pulumi.StringOutput),
})
if err != nil {
return err
}
return nil
})
}Content copied to clipboard
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azure.core.ResourceGroup;
import com.pulumi.azure.core.ResourceGroupArgs;
import com.pulumi.azure.datafactory.Factory;
import com.pulumi.azure.datafactory.FactoryArgs;
import com.pulumi.azure.databricks.Workspace;
import com.pulumi.azure.databricks.WorkspaceArgs;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricks;
import com.pulumi.azure.datafactory.LinkedServiceAzureDatabricksArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new ResourceGroup("example", ResourceGroupArgs.builder()
.name("example")
.location("East US")
.build());
//Link to an existing cluster via access token
var exampleFactory = new Factory("exampleFactory", FactoryArgs.builder()
.name("TestDtaFactory92783401247")
.location(example.location())
.resourceGroupName(example.name())
.build());
//Create a databricks instance
var exampleWorkspace = new Workspace("exampleWorkspace", WorkspaceArgs.builder()
.name("databricks-test")
.resourceGroupName(example.name())
.location(example.location())
.sku("standard")
.build());
var atLinked = new LinkedServiceAzureDatabricks("atLinked", LinkedServiceAzureDatabricksArgs.builder()
.name("ADBLinkedServiceViaAccessToken")
.dataFactoryId(exampleFactory.id())
.description("ADB Linked Service via Access Token")
.existingClusterId("0308-201146-sly615")
.accessToken("SomeDatabricksAccessToken")
.adbDomain(exampleWorkspace.workspaceUrl().applyValue(_workspaceUrl -> String.format("https://%s", _workspaceUrl)))
.build());
}
}Content copied to clipboard
resources:
example:
type: azure:core:ResourceGroup
properties:
name: example
location: East US
#Link to an existing cluster via access token
exampleFactory:
type: azure:datafactory:Factory
name: example
properties:
name: TestDtaFactory92783401247
location: ${example.location}
resourceGroupName: ${example.name}
#Create a databricks instance
exampleWorkspace:
type: azure:databricks:Workspace
name: example
properties:
name: databricks-test
resourceGroupName: ${example.name}
location: ${example.location}
sku: standard
atLinked:
type: azure:datafactory:LinkedServiceAzureDatabricks
name: at_linked
properties:
name: ADBLinkedServiceViaAccessToken
dataFactoryId: ${exampleFactory.id}
description: ADB Linked Service via Access Token
existingClusterId: 0308-201146-sly615
accessToken: SomeDatabricksAccessToken
adbDomain: https://${exampleWorkspace.workspaceUrl}Content copied to clipboard
Import
Data Factory Linked Services can be imported using the resource id, e.g.
$ pulumi import azure:datafactory/linkedServiceAzureDatabricks:LinkedServiceAzureDatabricks example /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/example/providers/Microsoft.DataFactory/factories/example/linkedservices/exampleContent copied to clipboard
Constructors
Link copied to clipboard
constructor(accessToken: Output<String>? = null, adbDomain: Output<String>? = null, additionalProperties: Output<Map<String, String>>? = null, annotations: Output<List<String>>? = null, dataFactoryId: Output<String>? = null, description: Output<String>? = null, existingClusterId: Output<String>? = null, instancePool: Output<LinkedServiceAzureDatabricksInstancePoolArgs>? = null, integrationRuntimeName: Output<String>? = null, keyVaultPassword: Output<LinkedServiceAzureDatabricksKeyVaultPasswordArgs>? = null, msiWorkSpaceResourceId: Output<String>? = null, name: Output<String>? = null, newClusterConfig: Output<LinkedServiceAzureDatabricksNewClusterConfigArgs>? = null, parameters: Output<Map<String, String>>? = null)
Properties
Link copied to clipboard
Authenticate to ADB via an access token.
Link copied to clipboard
A map of additional properties to associate with the Data Factory Linked Service.
Link copied to clipboard
List of tags that can be used for describing the Data Factory Linked Service.
Link copied to clipboard
The Data Factory ID in which to associate the Linked Service with. Changing this forces a new resource.
Link copied to clipboard
The description for the Data Factory Linked Service.
Link copied to clipboard
The cluster_id of an existing cluster within the linked ADB instance.
Link copied to clipboard
Leverages an instance pool within the linked ADB instance as one instance_pool block defined below.
Link copied to clipboard
The integration runtime reference to associate with the Data Factory Linked Service.
Link copied to clipboard
Authenticate to ADB via Azure Key Vault Linked Service as defined in the key_vault_password block below.
Link copied to clipboard
Authenticate to ADB via managed service identity.
Link copied to clipboard
Specifies the name of the Data Factory Linked Service. Changing this forces a new resource to be created. Must be unique within a data factory. See the Microsoft documentation for all restrictions.
Link copied to clipboard
Creates new clusters within the linked ADB instance as defined in the new_cluster_config block below.
Link copied to clipboard
A map of parameters to associate with the Data Factory Linked Service.