TestFeaturesConfig.java
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.sql.analyzer;
import com.facebook.airlift.configuration.ConfigurationFactory;
import com.facebook.airlift.configuration.testing.ConfigAssertions;
import com.facebook.presto.CompressionCodec;
import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationIfToFilterRewriteStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.CteMaterializationStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType;
import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.LeftJoinArrayContainsToInnerJoinStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.PartitioningPrecisionStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.PushDownFilterThroughCrossJoinStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.RandomizeOuterJoinNullKeyStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.SingleStreamSpillerChoice;
import com.google.common.collect.ImmutableMap;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import org.testng.annotations.Test;
import java.util.Map;
import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy.LEGACY;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy.TOP_DOWN;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinNotNullInferenceStrategy.USE_FUNCTION_METADATA;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.NONE;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.PartialMergePushdownStrategy.PUSH_THROUGH_LOW_MEMORY_OPERATORS;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.SPILLER_SPILL_PATH;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.SPILL_ENABLED;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.TaskSpillingStrategy.ORDER_BY_CREATE_TIME;
import static com.facebook.presto.sql.analyzer.FeaturesConfig.TaskSpillingStrategy.PER_TASK_MEMORY_THRESHOLD;
import static com.facebook.presto.sql.tree.CreateView.Security.DEFINER;
import static com.facebook.presto.sql.tree.CreateView.Security.INVOKER;
import static io.airlift.units.DataSize.Unit.GIGABYTE;
import static io.airlift.units.DataSize.Unit.KILOBYTE;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
public class TestFeaturesConfig
{
@Test
public void testDefaults()
{
assertRecordedDefaults(ConfigAssertions.recordDefaults(FeaturesConfig.class)
.setCpuCostWeight(75)
.setMemoryCostWeight(10)
.setNetworkCostWeight(15)
.setDistributedIndexJoinsEnabled(false)
.setJoinDistributionType(JoinDistributionType.AUTOMATIC)
.setJoinMaxBroadcastTableSize(new DataSize(100, MEGABYTE))
.setSizeBasedJoinDistributionTypeEnabled(true)
.setGroupedExecutionEnabled(true)
.setRecoverableGroupedExecutionEnabled(false)
.setMaxFailedTaskPercentage(0.3)
.setConcurrentLifespansPerTask(0)
.setFastInequalityJoins(true)
.setColocatedJoinsEnabled(true)
.setSpatialJoinsEnabled(true)
.setJoinReorderingStrategy(JoinReorderingStrategy.AUTOMATIC)
.setPartialMergePushdownStrategy(FeaturesConfig.PartialMergePushdownStrategy.NONE)
.setMaxReorderedJoins(9)
.setUseHistoryBasedPlanStatistics(false)
.setTrackHistoryBasedPlanStatistics(false)
.setTrackHistoryStatsFromFailedQuery(true)
.setUsePartialAggregationHistory(false)
.setTrackPartialAggregationHistory(true)
.setUsePerfectlyConsistentHistories(false)
.setHistoryCanonicalPlanNodeLimit(1000)
.setHistoryBasedOptimizerTimeout(new Duration(10, SECONDS))
.setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS")
.setLogPlansUsedInHistoryBasedOptimizer(false)
.setEnforceTimeoutForHBOQueryRegistration(false)
.setHistoryBasedOptimizerEstimateSizeUsingVariables(false)
.setRedistributeWrites(false)
.setScaleWriters(true)
.setWriterMinSize(new DataSize(32, MEGABYTE))
.setOptimizedScaleWriterProducerBuffer(true)
.setOptimizeMetadataQueries(false)
.setOptimizeMetadataQueriesIgnoreStats(false)
.setOptimizeMetadataQueriesCallThreshold(100)
.setOptimizeHashGeneration(true)
.setPushTableWriteThroughUnion(true)
.setDictionaryAggregation(false)
.setConfidenceBasedBroadcastEnabled(false)
.setRetryQueryWithHistoryBasedOptimizationEnabled(false)
.setTreatLowConfidenceZeroEstimationAsUnknownEnabled(false)
.setAggregationPartitioningMergingStrategy(LEGACY)
.setSpillEnabled(false)
.setJoinSpillingEnabled(true)
.setSpillerSpillPaths("")
.setSpillerThreads(4)
.setSpillMaxUsedSpaceThreshold(0.9)
.setMemoryRevokingThreshold(0.9)
.setMemoryRevokingTarget(0.5)
.setTaskSpillingStrategy(ORDER_BY_CREATE_TIME)
.setQueryLimitSpillEnabled(false)
.setSingleStreamSpillerChoice(SingleStreamSpillerChoice.LOCAL_FILE)
.setSpillerTempStorage("local")
.setMaxRevocableMemoryPerTask(new DataSize(500, MEGABYTE))
.setOptimizeMixedDistinctAggregations(false)
.setIterativeOptimizerEnabled(true)
.setIterativeOptimizerTimeout(new Duration(3, MINUTES))
.setRuntimeOptimizerEnabled(false)
.setEnableDynamicFiltering(false)
.setDynamicFilteringMaxPerDriverRowCount(100)
.setDynamicFilteringMaxPerDriverSize(new DataSize(10, KILOBYTE))
.setDynamicFilteringRangeRowLimitPerDriver(0)
.setFragmentResultCachingEnabled(false)
.setEnableStatsCalculator(true)
.setEnableStatsCollectionForTemporaryTable(false)
.setIgnoreStatsCalculatorFailures(true)
.setPrintStatsForNonJoinQuery(false)
.setDefaultFilterFactorEnabled(false)
.setExchangeCompressionCodec(CompressionCodec.NONE)
.setExchangeChecksumEnabled(false)
.setEnableIntermediateAggregations(false)
.setPushAggregationThroughJoin(true)
.setForceSingleNodeOutput(true)
.setPagesIndexEagerCompactionEnabled(false)
.setFilterAndProjectMinOutputPageSize(new DataSize(500, KILOBYTE))
.setFilterAndProjectMinOutputPageRowCount(256)
.setUseMarkDistinct(true)
.setExploitConstraints(true)
.setPreferPartialAggregation(true)
.setPartialAggregationStrategy(PartialAggregationStrategy.ALWAYS)
.setPartialAggregationByteReductionThreshold(0.5)
.setAdaptivePartialAggregationEnabled(false)
.setAdaptivePartialAggregationRowsReductionRatioThreshold(0.8)
.setOptimizeTopNRowNumber(true)
.setOptimizeCaseExpressionPredicate(false)
.setDistributedSortEnabled(true)
.setMaxGroupingSets(2048)
.setLegacyUnnestArrayRows(false)
.setJsonSerdeCodeGenerationEnabled(false)
.setPushLimitThroughOuterJoin(true)
.setOptimizeConstantGroupingKeys(true)
.setMaxConcurrentMaterializations(3)
.setPushdownSubfieldsEnabled(false)
.setPushdownSubfieldsFromLambdaEnabled(false)
.setPushdownDereferenceEnabled(false)
.setIndexLoaderTimeout(new Duration(20, SECONDS))
.setOptimizedRepartitioningEnabled(false)
.setListBuiltInFunctionsOnly(true)
.setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.AUTOMATIC)
.setExperimentalFunctionsEnabled(false)
.setOptimizeCommonSubExpressions(true)
.setPreferDistributedUnion(true)
.setOptimizeNullsInJoin(false)
.setOptimizePayloadJoins(false)
.setJoinsNotNullInferenceStrategy(FeaturesConfig.JoinNotNullInferenceStrategy.NONE)
.setSkipRedundantSort(true)
.setWarnOnNoTableLayoutFilter("")
.setInlineSqlFunctions(true)
.setCheckAccessControlOnUtilizedColumnsOnly(true)
.setCheckAccessControlWithSubfields(false)
.setAllowWindowOrderByLiterals(true)
.setEnforceFixedDistributionForOutputOperator(false)
.setEmptyJoinOptimization(false)
.setLogFormattedQueryEnabled(false)
.setLogInvokedFunctionNamesEnabled(false)
.setSpoolingOutputBufferEnabled(false)
.setSpoolingOutputBufferThreshold(new DataSize(8, MEGABYTE))
.setSpoolingOutputBufferTempStorage("local")
.setPrestoSparkAssignBucketToPartitionForPartitionedTableWriteEnabled(false)
.setPartialResultsEnabled(false)
.setPartialResultsCompletionRatioThreshold(0.5)
.setOffsetClauseEnabled(false)
.setPartialResultsMaxExecutionTimeMultiplier(2.0)
.setMaterializedViewDataConsistencyEnabled(true)
.setMaterializedViewPartitionFilteringEnabled(true)
.setQueryOptimizationWithMaterializedViewEnabled(false)
.setVerboseRuntimeStatsEnabled(false)
.setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.DISABLED)
.setAnalyzerType("BUILTIN")
.setPreProcessMetadataCalls(false)
.setStreamingForPartialAggregationEnabled(false)
.setMaxStageCountForEagerScheduling(25)
.setHyperloglogStandardErrorWarningThreshold(0.004)
.setPreferMergeJoinForSortedInputs(false)
.setSegmentedAggregationEnabled(false)
.setQueryAnalyzerTimeout(new Duration(3, MINUTES))
.setQuickDistinctLimitEnabled(false)
.setPushRemoteExchangeThroughGroupId(false)
.setOptimizeMultipleApproxPercentileOnSameFieldEnabled(true)
.setNativeExecutionEnabled(false)
.setDisableTimeStampWithTimeZoneForNative(false)
.setDisableIPAddressForNative(false)
.setNativeExecutionExecutablePath("./presto_server")
.setNativeExecutionProgramArguments("")
.setNativeExecutionProcessReuseEnabled(true)
.setNativeEnforceJoinBuildInputPartition(true)
.setRandomizeOuterJoinNullKeyEnabled(false)
.setRandomizeOuterJoinNullKeyStrategy(RandomizeOuterJoinNullKeyStrategy.DISABLED)
.setShardedJoinStrategy(FeaturesConfig.ShardedJoinStrategy.DISABLED)
.setJoinShardCount(100)
.setOptimizeConditionalAggregationEnabled(false)
.setRemoveRedundantDistinctAggregationEnabled(true)
.setInPredicatesAsInnerJoinsEnabled(false)
.setPushAggregationBelowJoinByteReductionThreshold(1)
.setPrefilterForGroupbyLimit(false)
.setOptimizeJoinProbeForEmptyBuildRuntimeEnabled(false)
.setUseDefaultsForCorrelatedAggregationPushdownThroughOuterJoins(true)
.setMergeDuplicateAggregationsEnabled(true)
.setMergeAggregationsWithAndWithoutFilter(false)
.setSimplifyPlanWithEmptyInput(true)
.setPushDownFilterExpressionEvaluationThroughCrossJoin(PushDownFilterThroughCrossJoinStrategy.REWRITTEN_TO_INNER_JOIN)
.setDefaultJoinSelectivityCoefficient(0)
.setRewriteCrossJoinWithOrFilterToInnerJoin(true)
.setRewriteCrossJoinWithArrayContainsFilterToInnerJoin(true)
.setLeftJoinWithArrayContainsToEquiJoinStrategy(LeftJoinArrayContainsToInnerJoinStrategy.DISABLED)
.setRewriteCrossJoinWithArrayNotContainsFilterToAntiJoin(true)
.setLeftJoinNullFilterToSemiJoin(true)
.setBroadcastJoinWithSmallBuildUnknownProbe(false)
.setAddPartialNodeForRowNumberWithLimitEnabled(true)
.setInferInequalityPredicates(false)
.setPullUpExpressionFromLambdaEnabled(false)
.setRewriteConstantArrayContainsToInEnabled(false)
.setUseHBOForScaledWriters(false)
.setRemoveRedundantCastToVarcharInJoin(true)
.setHandleComplexEquiJoins(false)
.setSkipHashGenerationForJoinWithTableScanInput(false)
.setCteMaterializationStrategy(CteMaterializationStrategy.NONE)
.setCteFilterAndProjectionPushdownEnabled(true)
.setGenerateDomainFilters(false)
.setRewriteExpressionWithConstantVariable(true)
.setOptimizeConditionalApproxDistinct(true)
.setDefaultWriterReplicationCoefficient(3.0)
.setDefaultViewSecurityMode(DEFINER)
.setCteHeuristicReplicationThreshold(4)
.setPrintEstimatedStatsFromCache(false)
.setRemoveCrossJoinWithSingleConstantRow(true)
.setUseHistograms(false)
.setInlineProjectionsOnValues(false)
.setIncludeValuesNodeInConnectorOptimizer(true)
.setEagerPlanValidationEnabled(false)
.setEagerPlanValidationThreadPoolSize(20)
.setPrestoSparkExecutionEnvironment(false)
.setSingleNodeExecutionEnabled(false)
.setNativeExecutionScaleWritersThreadsEnabled(false)
.setNativeExecutionTypeRewriteEnabled(false)
.setEnhancedCTESchedulingEnabled(true)
.setExpressionOptimizerName("default")
.setExcludeInvalidWorkerSessionProperties(false)
.setAddExchangeBelowPartialAggregationOverGroupId(false)
.setAddDistinctBelowSemiJoinBuild(false)
.setPushdownSubfieldForMapFunctions(true)
.setInnerJoinPushdownEnabled(false)
.setBroadcastSemiJoinForDelete(true)
.setInEqualityJoinPushdownEnabled(false)
.setRewriteMinMaxByToTopNEnabled(false)
.setPrestoSparkExecutionEnvironment(false));
}
@Test
public void testExplicitPropertyMappings()
{
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("cpu-cost-weight", "0.4")
.put("memory-cost-weight", "0.3")
.put("network-cost-weight", "0.2")
.put("experimental.iterative-optimizer-enabled", "false")
.put("experimental.iterative-optimizer-timeout", "10s")
.put("experimental.runtime-optimizer-enabled", "true")
.put("enable-dynamic-filtering", "true")
.put("dynamic-filtering-max-per-driver-row-count", "256")
.put("dynamic-filtering-max-per-driver-size", "64kB")
.put("dynamic-filtering-range-row-limit-per-driver", "1000")
.put("fragment-result-cache.enabled", "true")
.put("experimental.enable-stats-calculator", "false")
.put("experimental.enable-stats-collection-for-temporary-table", "true")
.put("optimizer.ignore-stats-calculator-failures", "false")
.put("print-stats-for-non-join-query", "true")
.put("optimizer.default-filter-factor-enabled", "true")
.put("distributed-index-joins-enabled", "true")
.put("join-distribution-type", "BROADCAST")
.put("join-max-broadcast-table-size", "42GB")
.put("optimizer.size-based-join-distribution-type-enabled", "false")
.put("grouped-execution-enabled", "false")
.put("recoverable-grouped-execution-enabled", "true")
.put("max-failed-task-percentage", "0.8")
.put("concurrent-lifespans-per-task", "1")
.put("fast-inequality-joins", "false")
.put("colocated-joins-enabled", "false")
.put("spatial-joins-enabled", "false")
.put("optimizer.join-reordering-strategy", "NONE")
.put("experimental.optimizer.partial-merge-pushdown-strategy", PUSH_THROUGH_LOW_MEMORY_OPERATORS.name())
.put("optimizer.max-reordered-joins", "5")
.put("optimizer.use-history-based-plan-statistics", "true")
.put("optimizer.track-history-based-plan-statistics", "true")
.put("optimizer.track-history-stats-from-failed-queries", "false")
.put("optimizer.use-partial-aggregation-history", "true")
.put("optimizer.track-partial-aggregation-history", "false")
.put("optimizer.use-perfectly-consistent-histories", "true")
.put("optimizer.history-canonical-plan-node-limit", "2")
.put("optimizer.history-based-optimizer-plan-canonicalization-strategies", "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")
.put("optimizer.log-plans-used-in-history-based-optimizer", "true")
.put("optimizer.enforce-timeout-for-hbo-query-registration", "true")
.put("optimizer.history-based-optimizer-estimate-size-using-variables", "true")
.put("optimizer.history-based-optimizer-timeout", "1s")
.put("redistribute-writes", "true")
.put("scale-writers", "false")
.put("writer-min-size", "42GB")
.put("optimized-scale-writer-producer-buffer", "false")
.put("optimizer.optimize-metadata-queries", "true")
.put("optimizer.optimize-metadata-queries-ignore-stats", "true")
.put("optimizer.optimize-metadata-queries-call-threshold", "200")
.put("optimizer.optimize-hash-generation", "false")
.put("optimizer.optimize-mixed-distinct-aggregations", "true")
.put("optimizer.push-table-write-through-union", "false")
.put("optimizer.dictionary-aggregation", "true")
.put("optimizer.confidence-based-broadcast", "true")
.put("optimizer.retry-query-with-history-based-optimization", "true")
.put("optimizer.treat-low-confidence-zero-estimation-as-unknown", "true")
.put("optimizer.push-aggregation-through-join", "false")
.put("optimizer.aggregation-partition-merging", "top_down")
.put("experimental.spill-enabled", "true")
.put("experimental.join-spill-enabled", "false")
.put("experimental.spiller-spill-path", "/tmp/custom/spill/path1,/tmp/custom/spill/path2")
.put("experimental.spiller-threads", "42")
.put("experimental.spiller-max-used-space-threshold", "0.8")
.put("experimental.memory-revoking-threshold", "0.2")
.put("experimental.memory-revoking-target", "0.8")
.put("experimental.spiller.task-spilling-strategy", "PER_TASK_MEMORY_THRESHOLD")
.put("experimental.query-limit-spill-enabled", "true")
.put("experimental.spiller.single-stream-spiller-choice", "TEMP_STORAGE")
.put("experimental.spiller.spiller-temp-storage", "crail")
.put("experimental.spiller.max-revocable-task-memory", "1GB")
.put("exchange.compression-codec", "LZ4")
.put("exchange.checksum-enabled", "true")
.put("optimizer.enable-intermediate-aggregations", "true")
.put("optimizer.force-single-node-output", "false")
.put("pages-index.eager-compaction-enabled", "true")
.put("experimental.filter-and-project-min-output-page-size", "1MB")
.put("experimental.filter-and-project-min-output-page-row-count", "2048")
.put("optimizer.use-mark-distinct", "false")
.put("optimizer.exploit-constraints", "false")
.put("optimizer.prefer-partial-aggregation", "false")
.put("optimizer.partial-aggregation-strategy", "automatic")
.put("optimizer.partial-aggregation-byte-reduction-threshold", "0.8")
.put("experimental.adaptive-partial-aggregation", "true")
.put("experimental.adaptive-partial-aggregation-rows-reduction-ratio-threshold", "0.9")
.put("optimizer.optimize-top-n-row-number", "false")
.put("optimizer.optimize-case-expression-predicate", "true")
.put("distributed-sort", "false")
.put("analyzer.max-grouping-sets", "2047")
.put("deprecated.legacy-unnest-array-rows", "true")
.put("experimental.json-serde-codegen-enabled", "true")
.put("optimizer.push-limit-through-outer-join", "false")
.put("optimizer.optimize-constant-grouping-keys", "false")
.put("max-concurrent-materializations", "5")
.put("experimental.pushdown-subfields-enabled", "true")
.put("pushdown-subfields-from-lambda-enabled", "true")
.put("experimental.pushdown-dereference-enabled", "true")
.put("index-loader-timeout", "10s")
.put("experimental.optimized-repartitioning", "true")
.put("list-built-in-functions-only", "false")
.put("partitioning-precision-strategy", "PREFER_EXACT_PARTITIONING")
.put("experimental-functions-enabled", "true")
.put("optimize-common-sub-expressions", "false")
.put("prefer-distributed-union", "false")
.put("optimize-nulls-in-join", "true")
.put("optimize-payload-joins", "true")
.put("optimizer.joins-not-null-inference-strategy", "USE_FUNCTION_METADATA")
.put("warn-on-no-table-layout-filter", "ry@nlikestheyankees,ds")
.put("inline-sql-functions", "false")
.put("check-access-control-on-utilized-columns-only", "false")
.put("check-access-control-with-subfields", "true")
.put("optimizer.skip-redundant-sort", "false")
.put("is-allow-window-order-by-literals", "false")
.put("enforce-fixed-distribution-for-output-operator", "true")
.put("optimizer.optimize-joins-with-empty-sources", "true")
.put("log-formatted-query-enabled", "true")
.put("log-invoked-function-names-enabled", "true")
.put("spooling-output-buffer-enabled", "true")
.put("spooling-output-buffer-threshold", "16MB")
.put("spooling-output-buffer-temp-storage", "tempfs")
.put("spark.assign-bucket-to-partition-for-partitioned-table-write-enabled", "true")
.put("partial-results-enabled", "true")
.put("partial-results-completion-ratio-threshold", "0.9")
.put("partial-results-max-execution-time-multiplier", "1.5")
.put("offset-clause-enabled", "true")
.put("materialized-view-data-consistency-enabled", "false")
.put("consider-query-filters-for-materialized-view-partitions", "false")
.put("query-optimization-with-materialized-view-enabled", "true")
.put("analyzer-type", "CRUX")
.put("pre-process-metadata-calls", "true")
.put("verbose-runtime-stats-enabled", "true")
.put("optimizer.aggregation-if-to-filter-rewrite-strategy", "filter_with_if")
.put("streaming-for-partial-aggregation-enabled", "true")
.put("execution-policy.max-stage-count-for-eager-scheduling", "123")
.put("hyperloglog-standard-error-warning-threshold", "0.02")
.put("optimizer.prefer-merge-join-for-sorted-inputs", "true")
.put("optimizer.segmented-aggregation-enabled", "true")
.put("planner.query-analyzer-timeout", "10s")
.put("optimizer.quick-distinct-limit-enabled", "true")
.put("optimizer.push-remote-exchange-through-group-id", "true")
.put("optimizer.optimize-multiple-approx-percentile-on-same-field", "false")
.put("native-execution-enabled", "true")
.put("disable-timestamp-with-timezone-for-native-execution", "true")
.put("disable-ipaddress-for-native-execution", "true")
.put("native-execution-executable-path", "/bin/echo")
.put("native-execution-program-arguments", "--v 1")
.put("native-execution-process-reuse-enabled", "false")
.put("native-enforce-join-build-input-partition", "false")
.put("optimizer.randomize-outer-join-null-key", "true")
.put("optimizer.randomize-outer-join-null-key-strategy", "key_from_outer_join")
.put("optimizer.sharded-join-strategy", "cost_based")
.put("optimizer.join-shard-count", "200")
.put("optimizer.optimize-conditional-aggregation-enabled", "true")
.put("optimizer.remove-redundant-distinct-aggregation-enabled", "false")
.put("optimizer.in-predicates-as-inner-joins-enabled", "true")
.put("optimizer.push-aggregation-below-join-byte-reduction-threshold", "0.9")
.put("optimizer.prefilter-for-groupby-limit", "true")
.put("optimizer.optimize-probe-for-empty-build-runtime", "true")
.put("optimizer.use-defaults-for-correlated-aggregation-pushdown-through-outer-joins", "false")
.put("optimizer.merge-duplicate-aggregations", "false")
.put("optimizer.merge-aggregations-with-and-without-filter", "true")
.put("optimizer.simplify-plan-with-empty-input", "false")
.put("optimizer.push-down-filter-expression-evaluation-through-cross-join", "DISABLED")
.put("optimizer.rewrite-cross-join-with-or-filter-to-inner-join", "false")
.put("optimizer.rewrite-cross-join-with-array-contains-filter-to-inner-join", "false")
.put("optimizer.left-join-with-array-contains-to-equi-join-strategy", "ALWAYS_ENABLED")
.put("optimizer.rewrite-cross-join-with-array-not-contains-filter-to-anti-join", "false")
.put("optimizer.default-join-selectivity-coefficient", "0.5")
.put("optimizer.rewrite-left-join-with-null-filter-to-semi-join", "false")
.put("experimental.optimizer.broadcast-join-with-small-build-unknown-probe", "true")
.put("optimizer.add-partial-node-for-row-number-with-limit", "false")
.put("optimizer.infer-inequality-predicates", "true")
.put("optimizer.pull-up-expression-from-lambda", "true")
.put("optimizer.rewrite-constant-array-contains-to-in", "true")
.put("optimizer.use-hbo-for-scaled-writers", "true")
.put("optimizer.remove-redundant-cast-to-varchar-in-join", "false")
.put("cte-materialization-strategy", "ALL")
.put("cte-filter-and-projection-pushdown-enabled", "false")
.put("optimizer.handle-complex-equi-joins", "true")
.put("optimizer.skip-hash-generation-for-join-with-table-scan-input", "true")
.put("optimizer.generate-domain-filters", "true")
.put("optimizer.rewrite-expression-with-constant-variable", "false")
.put("optimizer.optimize-constant-approx-distinct", "false")
.put("optimizer.default-writer-replication-coefficient", "5.0")
.put("default-view-security-mode", INVOKER.name())
.put("cte-heuristic-replication-threshold", "2")
.put("optimizer.print-estimated-stats-from-cache", "true")
.put("optimizer.remove-cross-join-with-single-constant-row", "false")
.put("optimizer.use-histograms", "true")
.put("optimizer.inline-projections-on-values", "true")
.put("optimizer.include-values-node-in-connector-optimizer", "false")
.put("eager-plan-validation-enabled", "true")
.put("eager-plan-validation-thread-pool-size", "2")
.put("optimizer.inner-join-pushdown-enabled", "true")
.put("optimizer.broadcast-semi-join-for-delete", "false")
.put("optimizer.inequality-join-pushdown-enabled", "true")
.put("optimizer.rewrite-minBy-maxBy-to-topN-enabled", "true")
.put("presto-spark-execution-environment", "true")
.put("single-node-execution-enabled", "true")
.put("native-execution-scale-writer-threads-enabled", "true")
.put("native-execution-type-rewrite-enabled", "true")
.put("enhanced-cte-scheduling-enabled", "false")
.put("expression-optimizer-name", "custom")
.put("exclude-invalid-worker-session-properties", "true")
.put("optimizer.add-distinct-below-semi-join-build", "true")
.put("optimizer.pushdown-subfield-for-map-functions", "false")
.put("optimizer.add-exchange-below-partial-aggregation-over-group-id", "true")
.build();
FeaturesConfig expected = new FeaturesConfig()
.setCpuCostWeight(0.4)
.setMemoryCostWeight(0.3)
.setNetworkCostWeight(0.2)
.setIterativeOptimizerEnabled(false)
.setIterativeOptimizerTimeout(new Duration(10, SECONDS))
.setRuntimeOptimizerEnabled(true)
.setEnableDynamicFiltering(true)
.setDynamicFilteringMaxPerDriverRowCount(256)
.setDynamicFilteringMaxPerDriverSize(new DataSize(64, KILOBYTE))
.setDynamicFilteringRangeRowLimitPerDriver(1000)
.setFragmentResultCachingEnabled(true)
.setEnableStatsCalculator(false)
.setEnableStatsCollectionForTemporaryTable(true)
.setIgnoreStatsCalculatorFailures(false)
.setPrintStatsForNonJoinQuery(true)
.setDistributedIndexJoinsEnabled(true)
.setJoinDistributionType(BROADCAST)
.setJoinMaxBroadcastTableSize(new DataSize(42, GIGABYTE))
.setSizeBasedJoinDistributionTypeEnabled(false)
.setGroupedExecutionEnabled(false)
.setRecoverableGroupedExecutionEnabled(true)
.setMaxFailedTaskPercentage(0.8)
.setConcurrentLifespansPerTask(1)
.setFastInequalityJoins(false)
.setColocatedJoinsEnabled(false)
.setSpatialJoinsEnabled(false)
.setJoinReorderingStrategy(NONE)
.setPartialMergePushdownStrategy(PUSH_THROUGH_LOW_MEMORY_OPERATORS)
.setMaxReorderedJoins(5)
.setUseHistoryBasedPlanStatistics(true)
.setTrackHistoryBasedPlanStatistics(true)
.setTrackHistoryStatsFromFailedQuery(false)
.setUsePartialAggregationHistory(true)
.setTrackPartialAggregationHistory(false)
.setUsePerfectlyConsistentHistories(true)
.setHistoryCanonicalPlanNodeLimit(2)
.setHistoryBasedOptimizerTimeout(new Duration(1, SECONDS))
.setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")
.setLogPlansUsedInHistoryBasedOptimizer(true)
.setEnforceTimeoutForHBOQueryRegistration(true)
.setHistoryBasedOptimizerEstimateSizeUsingVariables(true)
.setRedistributeWrites(true)
.setScaleWriters(false)
.setWriterMinSize(new DataSize(42, GIGABYTE))
.setOptimizedScaleWriterProducerBuffer(false)
.setOptimizeMetadataQueries(true)
.setOptimizeMetadataQueriesIgnoreStats(true)
.setOptimizeMetadataQueriesCallThreshold(200)
.setOptimizeHashGeneration(false)
.setOptimizeMixedDistinctAggregations(true)
.setPushTableWriteThroughUnion(false)
.setDictionaryAggregation(true)
.setConfidenceBasedBroadcastEnabled(true)
.setRetryQueryWithHistoryBasedOptimizationEnabled(true)
.setTreatLowConfidenceZeroEstimationAsUnknownEnabled(true)
.setAggregationPartitioningMergingStrategy(TOP_DOWN)
.setPushAggregationThroughJoin(false)
.setSpillEnabled(true)
.setJoinSpillingEnabled(false)
.setSpillerSpillPaths("/tmp/custom/spill/path1,/tmp/custom/spill/path2")
.setSpillerThreads(42)
.setSpillMaxUsedSpaceThreshold(0.8)
.setMemoryRevokingThreshold(0.2)
.setMemoryRevokingTarget(0.8)
.setTaskSpillingStrategy(PER_TASK_MEMORY_THRESHOLD)
.setQueryLimitSpillEnabled(true)
.setSingleStreamSpillerChoice(SingleStreamSpillerChoice.TEMP_STORAGE)
.setSpillerTempStorage("crail")
.setMaxRevocableMemoryPerTask(new DataSize(1, GIGABYTE))
.setExchangeCompressionCodec(CompressionCodec.LZ4)
.setExchangeChecksumEnabled(true)
.setEnableIntermediateAggregations(true)
.setForceSingleNodeOutput(false)
.setPagesIndexEagerCompactionEnabled(true)
.setFilterAndProjectMinOutputPageSize(new DataSize(1, MEGABYTE))
.setFilterAndProjectMinOutputPageRowCount(2048)
.setUseMarkDistinct(false)
.setExploitConstraints(false)
.setPreferPartialAggregation(false)
.setPartialAggregationStrategy(PartialAggregationStrategy.AUTOMATIC)
.setPartialAggregationByteReductionThreshold(0.8)
.setAdaptivePartialAggregationEnabled(true)
.setAdaptivePartialAggregationRowsReductionRatioThreshold(0.9)
.setOptimizeTopNRowNumber(false)
.setOptimizeCaseExpressionPredicate(true)
.setDistributedSortEnabled(false)
.setMaxGroupingSets(2047)
.setLegacyUnnestArrayRows(true)
.setDefaultFilterFactorEnabled(true)
.setJsonSerdeCodeGenerationEnabled(true)
.setPushLimitThroughOuterJoin(false)
.setOptimizeConstantGroupingKeys(false)
.setMaxConcurrentMaterializations(5)
.setPushdownSubfieldsEnabled(true)
.setPushdownSubfieldsFromLambdaEnabled(true)
.setPushdownDereferenceEnabled(true)
.setIndexLoaderTimeout(new Duration(10, SECONDS))
.setOptimizedRepartitioningEnabled(true)
.setListBuiltInFunctionsOnly(false)
.setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING)
.setExperimentalFunctionsEnabled(true)
.setOptimizeCommonSubExpressions(false)
.setPreferDistributedUnion(false)
.setOptimizeNullsInJoin(true)
.setOptimizePayloadJoins(true)
.setJoinsNotNullInferenceStrategy(USE_FUNCTION_METADATA)
.setSkipRedundantSort(false)
.setWarnOnNoTableLayoutFilter("ry@nlikestheyankees,ds")
.setInlineSqlFunctions(false)
.setCheckAccessControlOnUtilizedColumnsOnly(false)
.setCheckAccessControlWithSubfields(true)
.setSkipRedundantSort(false)
.setAllowWindowOrderByLiterals(false)
.setEnforceFixedDistributionForOutputOperator(true)
.setEmptyJoinOptimization(true)
.setLogFormattedQueryEnabled(true)
.setLogInvokedFunctionNamesEnabled(true)
.setSpoolingOutputBufferEnabled(true)
.setSpoolingOutputBufferThreshold(new DataSize(16, MEGABYTE))
.setSpoolingOutputBufferTempStorage("tempfs")
.setPrestoSparkAssignBucketToPartitionForPartitionedTableWriteEnabled(true)
.setPartialResultsEnabled(true)
.setPartialResultsCompletionRatioThreshold(0.9)
.setOffsetClauseEnabled(true)
.setPartialResultsMaxExecutionTimeMultiplier(1.5)
.setMaterializedViewDataConsistencyEnabled(false)
.setMaterializedViewPartitionFilteringEnabled(false)
.setQueryOptimizationWithMaterializedViewEnabled(true)
.setVerboseRuntimeStatsEnabled(true)
.setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.FILTER_WITH_IF)
.setAnalyzerType("CRUX")
.setPreProcessMetadataCalls(true)
.setStreamingForPartialAggregationEnabled(true)
.setMaxStageCountForEagerScheduling(123)
.setHyperloglogStandardErrorWarningThreshold(0.02)
.setPreferMergeJoinForSortedInputs(true)
.setSegmentedAggregationEnabled(true)
.setQueryAnalyzerTimeout(new Duration(10, SECONDS))
.setQuickDistinctLimitEnabled(true)
.setPushRemoteExchangeThroughGroupId(true)
.setOptimizeMultipleApproxPercentileOnSameFieldEnabled(false)
.setNativeExecutionEnabled(true)
.setDisableTimeStampWithTimeZoneForNative(true)
.setDisableIPAddressForNative(true)
.setNativeExecutionExecutablePath("/bin/echo")
.setNativeExecutionProgramArguments("--v 1")
.setNativeExecutionProcessReuseEnabled(false)
.setNativeEnforceJoinBuildInputPartition(false)
.setRandomizeOuterJoinNullKeyEnabled(true)
.setRandomizeOuterJoinNullKeyStrategy(RandomizeOuterJoinNullKeyStrategy.KEY_FROM_OUTER_JOIN)
.setShardedJoinStrategy(FeaturesConfig.ShardedJoinStrategy.COST_BASED)
.setJoinShardCount(200)
.setOptimizeConditionalAggregationEnabled(true)
.setRemoveRedundantDistinctAggregationEnabled(false)
.setInPredicatesAsInnerJoinsEnabled(true)
.setPushAggregationBelowJoinByteReductionThreshold(0.9)
.setPrefilterForGroupbyLimit(true)
.setOptimizeJoinProbeForEmptyBuildRuntimeEnabled(true)
.setUseDefaultsForCorrelatedAggregationPushdownThroughOuterJoins(false)
.setMergeDuplicateAggregationsEnabled(false)
.setMergeAggregationsWithAndWithoutFilter(true)
.setSimplifyPlanWithEmptyInput(false)
.setDefaultJoinSelectivityCoefficient(0.5)
.setPushDownFilterExpressionEvaluationThroughCrossJoin(PushDownFilterThroughCrossJoinStrategy.DISABLED)
.setRewriteCrossJoinWithOrFilterToInnerJoin(false)
.setRewriteCrossJoinWithArrayContainsFilterToInnerJoin(false)
.setLeftJoinWithArrayContainsToEquiJoinStrategy(LeftJoinArrayContainsToInnerJoinStrategy.ALWAYS_ENABLED)
.setRewriteCrossJoinWithArrayNotContainsFilterToAntiJoin(false)
.setLeftJoinNullFilterToSemiJoin(false)
.setBroadcastJoinWithSmallBuildUnknownProbe(true)
.setAddPartialNodeForRowNumberWithLimitEnabled(false)
.setInferInequalityPredicates(true)
.setPullUpExpressionFromLambdaEnabled(true)
.setRewriteConstantArrayContainsToInEnabled(true)
.setUseHBOForScaledWriters(true)
.setRemoveRedundantCastToVarcharInJoin(false)
.setHandleComplexEquiJoins(true)
.setSkipHashGenerationForJoinWithTableScanInput(true)
.setCteMaterializationStrategy(CteMaterializationStrategy.ALL)
.setCteFilterAndProjectionPushdownEnabled(false)
.setGenerateDomainFilters(true)
.setRewriteExpressionWithConstantVariable(false)
.setOptimizeConditionalApproxDistinct(false)
.setDefaultWriterReplicationCoefficient(5.0)
.setDefaultViewSecurityMode(INVOKER)
.setCteHeuristicReplicationThreshold(2)
.setPrintEstimatedStatsFromCache(true)
.setRemoveCrossJoinWithSingleConstantRow(false)
.setUseHistograms(true)
.setInlineProjectionsOnValues(true)
.setIncludeValuesNodeInConnectorOptimizer(false)
.setEagerPlanValidationEnabled(true)
.setEagerPlanValidationThreadPoolSize(2)
.setPrestoSparkExecutionEnvironment(true)
.setSingleNodeExecutionEnabled(true)
.setNativeExecutionScaleWritersThreadsEnabled(true)
.setNativeExecutionTypeRewriteEnabled(true)
.setEnhancedCTESchedulingEnabled(false)
.setExpressionOptimizerName("custom")
.setExcludeInvalidWorkerSessionProperties(true)
.setAddExchangeBelowPartialAggregationOverGroupId(true)
.setAddDistinctBelowSemiJoinBuild(true)
.setPushdownSubfieldForMapFunctions(false)
.setInEqualityJoinPushdownEnabled(true)
.setBroadcastSemiJoinForDelete(false)
.setRewriteMinMaxByToTopNEnabled(true)
.setInnerJoinPushdownEnabled(true)
.setPrestoSparkExecutionEnvironment(true);
assertFullMapping(properties, expected);
}
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*\\Q" + SPILLER_SPILL_PATH + " must be configured when " + SPILL_ENABLED + " is set to true\\E.*")
public void testValidateSpillConfiguredIfEnabled()
{
new ConfigurationFactory(ImmutableMap.of(SPILL_ENABLED, "true"))
.build(FeaturesConfig.class);
}
}