pulumi-gcp-kotlin/com.pulumi.gcp.datastream.kotlin/Stream

Stream

class Stream : KotlinCustomResource

A resource representing streaming data from a source to a destination. To get more information about Stream, see:

Example Usage

Datastream Stream Full

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.organizations.OrganizationsFunctions;
import com.pulumi.gcp.organizations.inputs.GetProjectArgs;
import com.pulumi.gcp.sql.DatabaseInstance;
import com.pulumi.gcp.sql.DatabaseInstanceArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsBackupConfigurationArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsIpConfigurationArgs;
import com.pulumi.gcp.sql.Database;
import com.pulumi.gcp.sql.DatabaseArgs;
import com.pulumi.random.RandomPassword;
import com.pulumi.random.RandomPasswordArgs;
import com.pulumi.gcp.sql.User;
import com.pulumi.gcp.sql.UserArgs;
import com.pulumi.gcp.datastream.ConnectionProfile;
import com.pulumi.gcp.datastream.ConnectionProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileMysqlProfileArgs;
import com.pulumi.gcp.storage.Bucket;
import com.pulumi.gcp.storage.BucketArgs;
import com.pulumi.gcp.storage.BucketIAMMember;
import com.pulumi.gcp.storage.BucketIAMMemberArgs;
import com.pulumi.gcp.kms.CryptoKeyIAMMember;
import com.pulumi.gcp.kms.CryptoKeyIAMMemberArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileGcsProfileArgs;
import com.pulumi.gcp.datastream.Stream;
import com.pulumi.gcp.datastream.StreamArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigMysqlSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigMysqlSourceConfigIncludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigMysqlSourceConfigExcludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigGcsDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigGcsDestinationConfigJsonFileFormatArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllMysqlExcludedObjectsArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        final var project = OrganizationsFunctions.getProject();
        var instance = new DatabaseInstance("instance", DatabaseInstanceArgs.builder()
            .databaseVersion("MYSQL_8_0")
            .region("us-central1")
            .settings(DatabaseInstanceSettingsArgs.builder()
                .tier("db-f1-micro")
                .backupConfiguration(DatabaseInstanceSettingsBackupConfigurationArgs.builder()
                    .enabled(true)
                    .binaryLogEnabled(true)
                    .build())
                .ipConfiguration(DatabaseInstanceSettingsIpConfigurationArgs.builder()
                    .authorizedNetworks(
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.71.242.81")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.28.29")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.6.157")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.234.134")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.239.218")
                            .build())
                    .build())
                .build())
            .deletionProtection(true)
            .build());
        var db = new Database("db", DatabaseArgs.builder()
            .instance(instance.name())
            .build());
        var pwd = new RandomPassword("pwd", RandomPasswordArgs.builder()
            .length(16)
            .special(false)
            .build());
        var user = new User("user", UserArgs.builder()
            .instance(instance.name())
            .host("%")
            .password(pwd.result())
            .build());
        var sourceConnectionProfile = new ConnectionProfile("sourceConnectionProfile", ConnectionProfileArgs.builder()
            .displayName("Source connection profile")
            .location("us-central1")
            .connectionProfileId("source-profile")
            .mysqlProfile(ConnectionProfileMysqlProfileArgs.builder()
                .hostname(instance.publicIpAddress())
                .username(user.name())
                .password(user.password())
                .build())
            .build());
        var bucket = new Bucket("bucket", BucketArgs.builder()
            .location("US")
            .uniformBucketLevelAccess(true)
            .build());
        var viewer = new BucketIAMMember("viewer", BucketIAMMemberArgs.builder()
            .bucket(bucket.name())
            .role("roles/storage.objectViewer")
            .member(String.format("serviceAccount:service-%s@gcp-sa-datastream.iam.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
            .build());
        var creator = new BucketIAMMember("creator", BucketIAMMemberArgs.builder()
            .bucket(bucket.name())
            .role("roles/storage.objectCreator")
            .member(String.format("serviceAccount:service-%s@gcp-sa-datastream.iam.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
            .build());
        var reader = new BucketIAMMember("reader", BucketIAMMemberArgs.builder()
            .bucket(bucket.name())
            .role("roles/storage.legacyBucketReader")
            .member(String.format("serviceAccount:service-%s@gcp-sa-datastream.iam.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
            .build());
        var keyUser = new CryptoKeyIAMMember("keyUser", CryptoKeyIAMMemberArgs.builder()
            .cryptoKeyId("kms-name")
            .role("roles/cloudkms.cryptoKeyEncrypterDecrypter")
            .member(String.format("serviceAccount:service-%s@gcp-sa-datastream.iam.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
            .build());
        var destinationConnectionProfile = new ConnectionProfile("destinationConnectionProfile", ConnectionProfileArgs.builder()
            .displayName("Connection profile")
            .location("us-central1")
            .connectionProfileId("destination-profile")
            .gcsProfile(ConnectionProfileGcsProfileArgs.builder()
                .bucket(bucket.name())
                .rootPath("/path")
                .build())
            .build());
        var default_ = new Stream("default", StreamArgs.builder()
            .streamId("my-stream")
            .desiredState("NOT_STARTED")
            .location("us-central1")
            .displayName("my stream")
            .labels(Map.of("key", "value"))
            .sourceConfig(StreamSourceConfigArgs.builder()
                .sourceConnectionProfile(sourceConnectionProfile.id())
                .mysqlSourceConfig(StreamSourceConfigMysqlSourceConfigArgs.builder()
                    .includeObjects(StreamSourceConfigMysqlSourceConfigIncludeObjectsArgs.builder()
                        .mysqlDatabases(StreamSourceConfigMysqlSourceConfigIncludeObjectsMysqlDatabaseArgs.builder()
                            .database("my-database")
                            .mysqlTables(StreamSourceConfigMysqlSourceConfigIncludeObjectsMysqlDatabaseMysqlTableArgs.builder()
                                .table("includedTable")
                                .mysqlColumns(StreamSourceConfigMysqlSourceConfigIncludeObjectsMysqlDatabaseMysqlTableMysqlColumnArgs.builder()
                                    .column("includedColumn")
                                    .dataType("VARCHAR")
                                    .collation("utf8mb4")
                                    .primaryKey(false)
                                    .nullable(false)
                                    .ordinalPosition(0)
                                    .build())
                                .build())
                            .build())
                        .build())
                    .excludeObjects(StreamSourceConfigMysqlSourceConfigExcludeObjectsArgs.builder()
                        .mysqlDatabases(StreamSourceConfigMysqlSourceConfigExcludeObjectsMysqlDatabaseArgs.builder()
                            .database("my-database")
                            .mysqlTables(StreamSourceConfigMysqlSourceConfigExcludeObjectsMysqlDatabaseMysqlTableArgs.builder()
                                .table("excludedTable")
                                .mysqlColumns(StreamSourceConfigMysqlSourceConfigExcludeObjectsMysqlDatabaseMysqlTableMysqlColumnArgs.builder()
                                    .column("excludedColumn")
                                    .dataType("VARCHAR")
                                    .collation("utf8mb4")
                                    .primaryKey(false)
                                    .nullable(false)
                                    .ordinalPosition(0)
                                    .build())
                                .build())
                            .build())
                        .build())
                    .maxConcurrentCdcTasks(5)
                    .build())
                .build())
            .destinationConfig(StreamDestinationConfigArgs.builder()
                .destinationConnectionProfile(destinationConnectionProfile.id())
                .gcsDestinationConfig(StreamDestinationConfigGcsDestinationConfigArgs.builder()
                    .path("mydata")
                    .fileRotationMb(200)
                    .fileRotationInterval("60s")
                    .jsonFileFormat(StreamDestinationConfigGcsDestinationConfigJsonFileFormatArgs.builder()
                        .schemaFileFormat("NO_SCHEMA_FILE")
                        .compression("GZIP")
                        .build())
                    .build())
                .build())
            .backfillAll(StreamBackfillAllArgs.builder()
                .mysqlExcludedObjects(StreamBackfillAllMysqlExcludedObjectsArgs.builder()
                    .mysqlDatabases(StreamBackfillAllMysqlExcludedObjectsMysqlDatabaseArgs.builder()
                        .database("my-database")
                        .mysqlTables(StreamBackfillAllMysqlExcludedObjectsMysqlDatabaseMysqlTableArgs.builder()
                            .table("excludedTable")
                            .mysqlColumns(StreamBackfillAllMysqlExcludedObjectsMysqlDatabaseMysqlTableMysqlColumnArgs.builder()
                                .column("excludedColumn")
                                .dataType("VARCHAR")
                                .collation("utf8mb4")
                                .primaryKey(false)
                                .nullable(false)
                                .ordinalPosition(0)
                                .build())
                            .build())
                        .build())
                    .build())
                .build())
            .customerManagedEncryptionKey("kms-name")
            .build(), CustomResourceOptions.builder()
                .dependsOn(keyUser)
                .build());
    }
}
Content copied to clipboard

Datastream Stream Postgresql

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.datastream.ConnectionProfile;
import com.pulumi.gcp.datastream.ConnectionProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfilePostgresqlProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileBigqueryProfileArgs;
import com.pulumi.gcp.datastream.Stream;
import com.pulumi.gcp.datastream.StreamArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigPostgresqlSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigPostgresqlSourceConfigIncludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigPostgresqlSourceConfigExcludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllPostgresqlExcludedObjectsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var source = new ConnectionProfile("source", ConnectionProfileArgs.builder()
            .displayName("Postgresql Source")
            .location("us-central1")
            .connectionProfileId("source-profile")
            .postgresqlProfile(ConnectionProfilePostgresqlProfileArgs.builder()
                .hostname("hostname")
                .port(3306)
                .username("user")
                .password("pass")
                .database("postgres")
                .build())
            .build());
        var destination = new ConnectionProfile("destination", ConnectionProfileArgs.builder()
            .displayName("BigQuery Destination")
            .location("us-central1")
            .connectionProfileId("destination-profile")
            .bigqueryProfile()
            .build());
        var default_ = new Stream("default", StreamArgs.builder()
            .displayName("Postgres to BigQuery")
            .location("us-central1")
            .streamId("my-stream")
            .desiredState("RUNNING")
            .sourceConfig(StreamSourceConfigArgs.builder()
                .sourceConnectionProfile(source.id())
                .postgresqlSourceConfig(StreamSourceConfigPostgresqlSourceConfigArgs.builder()
                    .maxConcurrentBackfillTasks(12)
                    .publication("publication")
                    .replicationSlot("replication_slot")
                    .includeObjects(StreamSourceConfigPostgresqlSourceConfigIncludeObjectsArgs.builder()
                        .postgresqlSchemas(StreamSourceConfigPostgresqlSourceConfigIncludeObjectsPostgresqlSchemaArgs.builder()
                            .schema("schema")
                            .postgresqlTables(StreamSourceConfigPostgresqlSourceConfigIncludeObjectsPostgresqlSchemaPostgresqlTableArgs.builder()
                                .table("table")
                                .postgresqlColumns(StreamSourceConfigPostgresqlSourceConfigIncludeObjectsPostgresqlSchemaPostgresqlTablePostgresqlColumnArgs.builder()
                                    .column("column")
                                    .build())
                                .build())
                            .build())
                        .build())
                    .excludeObjects(StreamSourceConfigPostgresqlSourceConfigExcludeObjectsArgs.builder()
                        .postgresqlSchemas(StreamSourceConfigPostgresqlSourceConfigExcludeObjectsPostgresqlSchemaArgs.builder()
                            .schema("schema")
                            .postgresqlTables(StreamSourceConfigPostgresqlSourceConfigExcludeObjectsPostgresqlSchemaPostgresqlTableArgs.builder()
                                .table("table")
                                .postgresqlColumns(StreamSourceConfigPostgresqlSourceConfigExcludeObjectsPostgresqlSchemaPostgresqlTablePostgresqlColumnArgs.builder()
                                    .column("column")
                                    .build())
                                .build())
                            .build())
                        .build())
                    .build())
                .build())
            .destinationConfig(StreamDestinationConfigArgs.builder()
                .destinationConnectionProfile(destination.id())
                .bigqueryDestinationConfig(StreamDestinationConfigBigqueryDestinationConfigArgs.builder()
                    .dataFreshness("900s")
                    .sourceHierarchyDatasets(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs.builder()
                        .datasetTemplate(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs.builder()
                            .location("us-central1")
                            .build())
                        .build())
                    .build())
                .build())
            .backfillAll(StreamBackfillAllArgs.builder()
                .postgresqlExcludedObjects(StreamBackfillAllPostgresqlExcludedObjectsArgs.builder()
                    .postgresqlSchemas(StreamBackfillAllPostgresqlExcludedObjectsPostgresqlSchemaArgs.builder()
                        .schema("schema")
                        .postgresqlTables(StreamBackfillAllPostgresqlExcludedObjectsPostgresqlSchemaPostgresqlTableArgs.builder()
                            .table("table")
                            .postgresqlColumns(StreamBackfillAllPostgresqlExcludedObjectsPostgresqlSchemaPostgresqlTablePostgresqlColumnArgs.builder()
                                .column("column")
                                .build())
                            .build())
                        .build())
                    .build())
                .build())
            .build());
    }
}
Content copied to clipboard

Datastream Stream Oracle

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.datastream.ConnectionProfile;
import com.pulumi.gcp.datastream.ConnectionProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileOracleProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileBigqueryProfileArgs;
import com.pulumi.gcp.datastream.Stream;
import com.pulumi.gcp.datastream.StreamArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigOracleSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigOracleSourceConfigIncludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigOracleSourceConfigExcludeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigOracleSourceConfigDropLargeObjectsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllOracleExcludedObjectsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var source = new ConnectionProfile("source", ConnectionProfileArgs.builder()
            .displayName("Oracle Source")
            .location("us-central1")
            .connectionProfileId("source-profile")
            .oracleProfile(ConnectionProfileOracleProfileArgs.builder()
                .hostname("hostname")
                .port(1521)
                .username("user")
                .password("pass")
                .databaseService("ORCL")
                .build())
            .build());
        var destination = new ConnectionProfile("destination", ConnectionProfileArgs.builder()
            .displayName("BigQuery Destination")
            .location("us-central1")
            .connectionProfileId("destination-profile")
            .bigqueryProfile()
            .build());
        var stream5 = new Stream("stream5", StreamArgs.builder()
            .displayName("Oracle to BigQuery")
            .location("us-central1")
            .streamId("my-stream")
            .desiredState("RUNNING")
            .sourceConfig(StreamSourceConfigArgs.builder()
                .sourceConnectionProfile(source.id())
                .oracleSourceConfig(StreamSourceConfigOracleSourceConfigArgs.builder()
                    .maxConcurrentCdcTasks(8)
                    .maxConcurrentBackfillTasks(12)
                    .includeObjects(StreamSourceConfigOracleSourceConfigIncludeObjectsArgs.builder()
                        .oracleSchemas(StreamSourceConfigOracleSourceConfigIncludeObjectsOracleSchemaArgs.builder()
                            .schema("schema")
                            .oracleTables(StreamSourceConfigOracleSourceConfigIncludeObjectsOracleSchemaOracleTableArgs.builder()
                                .table("table")
                                .oracleColumns(StreamSourceConfigOracleSourceConfigIncludeObjectsOracleSchemaOracleTableOracleColumnArgs.builder()
                                    .column("column")
                                    .build())
                                .build())
                            .build())
                        .build())
                    .excludeObjects(StreamSourceConfigOracleSourceConfigExcludeObjectsArgs.builder()
                        .oracleSchemas(StreamSourceConfigOracleSourceConfigExcludeObjectsOracleSchemaArgs.builder()
                            .schema("schema")
                            .oracleTables(StreamSourceConfigOracleSourceConfigExcludeObjectsOracleSchemaOracleTableArgs.builder()
                                .table("table")
                                .oracleColumns(StreamSourceConfigOracleSourceConfigExcludeObjectsOracleSchemaOracleTableOracleColumnArgs.builder()
                                    .column("column")
                                    .build())
                                .build())
                            .build())
                        .build())
                    .dropLargeObjects()
                    .build())
                .build())
            .destinationConfig(StreamDestinationConfigArgs.builder()
                .destinationConnectionProfile(destination.id())
                .bigqueryDestinationConfig(StreamDestinationConfigBigqueryDestinationConfigArgs.builder()
                    .dataFreshness("900s")
                    .sourceHierarchyDatasets(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs.builder()
                        .datasetTemplate(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs.builder()
                            .location("us-central1")
                            .build())
                        .build())
                    .build())
                .build())
            .backfillAll(StreamBackfillAllArgs.builder()
                .oracleExcludedObjects(StreamBackfillAllOracleExcludedObjectsArgs.builder()
                    .oracleSchemas(StreamBackfillAllOracleExcludedObjectsOracleSchemaArgs.builder()
                        .schema("schema")
                        .oracleTables(StreamBackfillAllOracleExcludedObjectsOracleSchemaOracleTableArgs.builder()
                            .table("table")
                            .oracleColumns(StreamBackfillAllOracleExcludedObjectsOracleSchemaOracleTableOracleColumnArgs.builder()
                                .column("column")
                                .build())
                            .build())
                        .build())
                    .build())
                .build())
            .build());
    }
}
Content copied to clipboard

Datastream Stream Postgresql Bigquery Dataset Id

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Dataset;
import com.pulumi.gcp.bigquery.DatasetArgs;
import com.pulumi.gcp.datastream.ConnectionProfile;
import com.pulumi.gcp.datastream.ConnectionProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileBigqueryProfileArgs;
import com.pulumi.gcp.sql.DatabaseInstance;
import com.pulumi.gcp.sql.DatabaseInstanceArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsBackupConfigurationArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsIpConfigurationArgs;
import com.pulumi.random.RandomPassword;
import com.pulumi.random.RandomPasswordArgs;
import com.pulumi.gcp.sql.User;
import com.pulumi.gcp.sql.UserArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileMysqlProfileArgs;
import com.pulumi.gcp.datastream.Stream;
import com.pulumi.gcp.datastream.StreamArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigMysqlSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSingleTargetDatasetArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillAllArgs;
import com.pulumi.gcp.sql.Database;
import com.pulumi.gcp.sql.DatabaseArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        var postgres = new Dataset("postgres", DatasetArgs.builder()
            .datasetId("postgres")
            .friendlyName("postgres")
            .description("Database of postgres")
            .location("us-central1")
            .build());
        var destinationConnectionProfile2 = new ConnectionProfile("destinationConnectionProfile2", ConnectionProfileArgs.builder()
            .displayName("Connection profile")
            .location("us-central1")
            .connectionProfileId("dest-profile")
            .bigqueryProfile()
            .build());
        var instance = new DatabaseInstance("instance", DatabaseInstanceArgs.builder()
            .databaseVersion("MYSQL_8_0")
            .region("us-central1")
            .settings(DatabaseInstanceSettingsArgs.builder()
                .tier("db-f1-micro")
                .backupConfiguration(DatabaseInstanceSettingsBackupConfigurationArgs.builder()
                    .enabled(true)
                    .binaryLogEnabled(true)
                    .build())
                .ipConfiguration(DatabaseInstanceSettingsIpConfigurationArgs.builder()
                    .authorizedNetworks(
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.71.242.81")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.28.29")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.6.157")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.234.134")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.239.218")
                            .build())
                    .build())
                .build())
            .deletionProtection(false)
            .build());
        var pwd = new RandomPassword("pwd", RandomPasswordArgs.builder()
            .length(16)
            .special(false)
            .build());
        var user = new User("user", UserArgs.builder()
            .instance(instance.name())
            .host("%")
            .password(pwd.result())
            .build());
        var sourceConnectionProfile = new ConnectionProfile("sourceConnectionProfile", ConnectionProfileArgs.builder()
            .displayName("Source connection profile")
            .location("us-central1")
            .connectionProfileId("source-profile")
            .mysqlProfile(ConnectionProfileMysqlProfileArgs.builder()
                .hostname(instance.publicIpAddress())
                .username(user.name())
                .password(user.password())
                .build())
            .build());
        var default_ = new Stream("default", StreamArgs.builder()
            .displayName("postgres to bigQuery")
            .location("us-central1")
            .streamId("postgres-bigquery")
            .sourceConfig(StreamSourceConfigArgs.builder()
                .sourceConnectionProfile(sourceConnectionProfile.id())
                .mysqlSourceConfig()
                .build())
            .destinationConfig(StreamDestinationConfigArgs.builder()
                .destinationConnectionProfile(destinationConnectionProfile2.id())
                .bigqueryDestinationConfig(StreamDestinationConfigBigqueryDestinationConfigArgs.builder()
                    .dataFreshness("900s")
                    .singleTargetDataset(StreamDestinationConfigBigqueryDestinationConfigSingleTargetDatasetArgs.builder()
                        .datasetId(postgres.id())
                        .build())
                    .build())
                .build())
            .backfillAll()
            .build());
        var db = new Database("db", DatabaseArgs.builder()
            .instance(instance.name())
            .build());
    }
}
Content copied to clipboard

Datastream Stream Bigquery

package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.organizations.OrganizationsFunctions;
import com.pulumi.gcp.organizations.inputs.GetProjectArgs;
import com.pulumi.gcp.sql.DatabaseInstance;
import com.pulumi.gcp.sql.DatabaseInstanceArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsBackupConfigurationArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsIpConfigurationArgs;
import com.pulumi.gcp.sql.Database;
import com.pulumi.gcp.sql.DatabaseArgs;
import com.pulumi.random.RandomPassword;
import com.pulumi.random.RandomPasswordArgs;
import com.pulumi.gcp.sql.User;
import com.pulumi.gcp.sql.UserArgs;
import com.pulumi.gcp.datastream.ConnectionProfile;
import com.pulumi.gcp.datastream.ConnectionProfileArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileMysqlProfileArgs;
import com.pulumi.gcp.bigquery.BigqueryFunctions;
import com.pulumi.gcp.bigquery.inputs.GetDefaultServiceAccountArgs;
import com.pulumi.gcp.kms.CryptoKeyIAMMember;
import com.pulumi.gcp.kms.CryptoKeyIAMMemberArgs;
import com.pulumi.gcp.datastream.inputs.ConnectionProfileBigqueryProfileArgs;
import com.pulumi.gcp.datastream.Stream;
import com.pulumi.gcp.datastream.StreamArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamSourceConfigMysqlSourceConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs;
import com.pulumi.gcp.datastream.inputs.StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs;
import com.pulumi.gcp.datastream.inputs.StreamBackfillNoneArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }
    public static void stack(Context ctx) {
        final var project = OrganizationsFunctions.getProject();
        var instance = new DatabaseInstance("instance", DatabaseInstanceArgs.builder()
            .databaseVersion("MYSQL_8_0")
            .region("us-central1")
            .settings(DatabaseInstanceSettingsArgs.builder()
                .tier("db-f1-micro")
                .backupConfiguration(DatabaseInstanceSettingsBackupConfigurationArgs.builder()
                    .enabled(true)
                    .binaryLogEnabled(true)
                    .build())
                .ipConfiguration(DatabaseInstanceSettingsIpConfigurationArgs.builder()
                    .authorizedNetworks(
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.71.242.81")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.28.29")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.6.157")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.67.234.134")
                            .build(),
                        DatabaseInstanceSettingsIpConfigurationAuthorizedNetworkArgs.builder()
                            .value("34.72.239.218")
                            .build())
                    .build())
                .build())
            .deletionProtection(true)
            .build());
        var db = new Database("db", DatabaseArgs.builder()
            .instance(instance.name())
            .build());
        var pwd = new RandomPassword("pwd", RandomPasswordArgs.builder()
            .length(16)
            .special(false)
            .build());
        var user = new User("user", UserArgs.builder()
            .instance(instance.name())
            .host("%")
            .password(pwd.result())
            .build());
        var sourceConnectionProfile = new ConnectionProfile("sourceConnectionProfile", ConnectionProfileArgs.builder()
            .displayName("Source connection profile")
            .location("us-central1")
            .connectionProfileId("source-profile")
            .mysqlProfile(ConnectionProfileMysqlProfileArgs.builder()
                .hostname(instance.publicIpAddress())
                .username(user.name())
                .password(user.password())
                .build())
            .build());
        final var bqSa = BigqueryFunctions.getDefaultServiceAccount();
        var bigqueryKeyUser = new CryptoKeyIAMMember("bigqueryKeyUser", CryptoKeyIAMMemberArgs.builder()
            .cryptoKeyId("bigquery-kms-name")
            .role("roles/cloudkms.cryptoKeyEncrypterDecrypter")
            .member(String.format("serviceAccount:%s", bqSa.applyValue(getDefaultServiceAccountResult -> getDefaultServiceAccountResult.email())))
            .build());
        var destinationConnectionProfile = new ConnectionProfile("destinationConnectionProfile", ConnectionProfileArgs.builder()
            .displayName("Connection profile")
            .location("us-central1")
            .connectionProfileId("destination-profile")
            .bigqueryProfile()
            .build());
        var default_ = new Stream("default", StreamArgs.builder()
            .streamId("my-stream")
            .location("us-central1")
            .displayName("my stream")
            .sourceConfig(StreamSourceConfigArgs.builder()
                .sourceConnectionProfile(sourceConnectionProfile.id())
                .mysqlSourceConfig()
                .build())
            .destinationConfig(StreamDestinationConfigArgs.builder()
                .destinationConnectionProfile(destinationConnectionProfile.id())
                .bigqueryDestinationConfig(StreamDestinationConfigBigqueryDestinationConfigArgs.builder()
                    .sourceHierarchyDatasets(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsArgs.builder()
                        .datasetTemplate(StreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateArgs.builder()
                            .location("us-central1")
                            .kmsKeyName("bigquery-kms-name")
                            .build())
                        .build())
                    .build())
                .build())
            .backfillNone()
            .build(), CustomResourceOptions.builder()
                .dependsOn(bigqueryKeyUser)
                .build());
    }
}
Content copied to clipboard

Import

Stream can be imported using any of these accepted formats

$ pulumi import gcp:datastream/stream:Stream default projects/{{project}}/locations/{{location}}/streams/{{stream_id}}
Content copied to clipboard
$ pulumi import gcp:datastream/stream:Stream default {{project}}/{{location}}/{{stream_id}}
Content copied to clipboard
$ pulumi import gcp:datastream/stream:Stream default {{location}}/{{stream_id}}
Content copied to clipboard

Properties

Link copied to clipboard
val backfillAll: Output<StreamBackfillAll>?

Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. Structure is documented below.

Link copied to clipboard
val backfillNone: Output<StreamBackfillNone>?

Backfill strategy to disable automatic backfill for the Stream's objects.

Link copied to clipboard
val customerManagedEncryptionKey: Output<String>?

A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data will be encrypted using an internal Stream-specific encryption key provisioned through KMS.

Link copied to clipboard
val desiredState: Output<String>?

Desired state of the Stream. Set this field to RUNNING to start the stream, and PAUSED to pause the stream.

Link copied to clipboard
val destinationConfig: Output<StreamDestinationConfig>

Destination connection profile configuration. Structure is documented below.

Link copied to clipboard
val displayName: Output<String>

Display name.

Link copied to clipboard
val id: Output<String>
Link copied to clipboard
val labels: Output<Map<String, String>>?

Labels.

Link copied to clipboard
val location: Output<String>

The name of the location this stream is located in.

Link copied to clipboard
val name: Output<String>

The stream's name.

Link copied to clipboard
val project: Output<String>

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

Link copied to clipboard
val pulumiChildResources: Set<KotlinResource>
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
val sourceConfig: Output<StreamSourceConfig>

Source connection profile configuration. Structure is documented below.

Link copied to clipboard
val state: Output<String>

The state of the stream.

Link copied to clipboard
val streamId: Output<String>

The stream identifier.

Link copied to clipboard
val urn: Output<String>