Skip to content

Commit

Permalink
fix typo :facepalm
Browse files Browse the repository at this point in the history
  • Loading branch information
@alamb
alamb committed Oct 2, 2023
1 parent 177c529 commit d99d884
Show file tree
Hide file tree
Showing 6 changed files with 11 additions and 11 deletions.
2 changes: 1 addition & 1 deletion datafusion/common/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,7 @@ config_namespace! {
///
/// When false, DataFusion will prefer to maximize the parallelism using
/// `Repartition/Coalesce` and resort the data subsequently with `SortExec`
pub prefer_exising_sort: bool, default = false
pub prefer_existing_sort: bool, default = false

/// When set to true, the logical plan optimizer will produce warning
/// messages if any optimization rules produce errors and then proceed to the next
Expand Down
10 changes: 5 additions & 5 deletions datafusion/core/src/physical_optimizer/enforce_distribution.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1228,7 +1228,7 @@ fn ensure_distribution(
// - it is desired according to config
// - when plan is unbounded
let order_preserving_variants_desirable =
is_unbounded || config.optimizer.prefer_exising_sort;
is_unbounded || config.optimizer.prefer_existing_sort;

if dist_context.plan.children().is_empty() {
return Ok(Transformed::No(dist_context));
Expand Down Expand Up @@ -2085,7 +2085,7 @@ mod tests {
config.optimizer.enable_round_robin_repartition = false;
config.optimizer.repartition_file_scans = false;
config.optimizer.repartition_file_min_size = 1024;
config.optimizer.prefer_exising_sort = bounded_order_preserving_variants;
config.optimizer.prefer_existing_sort = bounded_order_preserving_variants;
ensure_distribution(distribution_context, &config).map(|item| item.into().plan)
}

Expand Down Expand Up @@ -2123,7 +2123,7 @@ mod tests {
config.execution.target_partitions = $TARGET_PARTITIONS;
config.optimizer.repartition_file_scans = $REPARTITION_FILE_SCANS;
config.optimizer.repartition_file_min_size = $REPARTITION_FILE_MIN_SIZE;
config.optimizer.prefer_exising_sort = $BOUNDED_ORDER_PRESERVING_VARIANTS;
config.optimizer.prefer_existing_sort = $BOUNDED_ORDER_PRESERVING_VARIANTS;

// NOTE: These tests verify the joint `EnforceDistribution` + `EnforceSorting` cascade
// because they were written prior to the separation of `BasicEnforcement` into
Expand Down Expand Up @@ -4515,7 +4515,7 @@ mod tests {
let mut config = ConfigOptions::new();
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_exising_sort = false;
config.optimizer.prefer_existing_sort = false;
let distribution_plan =
EnforceDistribution::new().optimize(physical_plan, &config)?;
assert_plan_txt!(expected, distribution_plan);
Expand Down Expand Up @@ -4557,7 +4557,7 @@ mod tests {
let mut config = ConfigOptions::new();
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_exising_sort = false;
config.optimizer.prefer_existing_sort = false;
let distribution_plan =
EnforceDistribution::new().optimize(physical_plan, &config)?;
assert_plan_txt!(expected, distribution_plan);
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ pub(crate) fn replace_with_order_preserving_variants(
// any case, as doing so helps fix the pipeline.
// Also do the replacement if opted-in via config options.
let use_order_preserving_variant =
config.optimizer.prefer_exising_sort || unbounded_output(plan);
config.optimizer.prefer_existing_sort || unbounded_output(plan);
let updated_sort_input = get_updated_plan(
exec_tree,
is_spr_better || use_order_preserving_variant,
Expand Down
4 changes: 2 additions & 2 deletions datafusion/execution/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ impl SessionConfig {
///
/// [prefer_existing_sort]: datafusion_common::config::OptimizerOptions::prefer_existing_sort
pub fn prefer_existing_sort(&self) -> bool {
self.options.optimizer.prefer_exising_sort
self.options.optimizer.prefer_existing_sort
}

/// Are statistics collected during execution?
Expand Down Expand Up @@ -228,7 +228,7 @@ impl SessionConfig {
///
/// [prefer_existing_sort]: datafusion_common::config::OptimizerOptions::prefer_existing_sort
pub fn with_prefer_existing_sort(mut self, enabled: bool) -> Self {
self.options.optimizer.prefer_exising_sort = enabled;
self.options.optimizer.prefer_existing_sort = enabled;
self
}

Expand Down
2 changes: 1 addition & 1 deletion datafusion/sqllogictest/test_files/information_schema.slt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ datafusion.optimizer.enable_topk_aggregation true
datafusion.optimizer.filter_null_join_keys false
datafusion.optimizer.hash_join_single_partition_threshold 1048576
datafusion.optimizer.max_passes 3
datafusion.optimizer.prefer_exising_sort false
datafusion.optimizer.prefer_existing_sort false
datafusion.optimizer.prefer_hash_join true
datafusion.optimizer.repartition_aggregations true
datafusion.optimizer.repartition_file_min_size 10485760
Expand Down
2 changes: 1 addition & 1 deletion docs/source/user-guide/configs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus
| datafusion.optimizer.repartition_file_scans | true | When set to `true`, file groups will be repartitioned to achieve maximum parallelism. Currently Parquet and CSV formats are supported. If set to `true`, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false`, different files will be read in parallel, but repartitioning won't happen within a single file. |
| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level |
| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` |
| datafusion.optimizer.prefer_exising_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, replacing `RepartitionExec` with `SortPreservingRepartitionExec`, and `CoalescePartitionsExec` with `SortPreservingMergeExec`, When false, DataFusion will prefer to maximize the parallelism using `Repartition/Coalesce` and resort the data subsequently with `SortExec` |
| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, replacing `RepartitionExec` with `SortPreservingRepartitionExec`, and `CoalescePartitionsExec` with `SortPreservingMergeExec`, When false, DataFusion will prefer to maximize the parallelism using `Repartition/Coalesce` and resort the data subsequently with `SortExec` |
| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail |
| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan |
| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys |
Expand Down

0 comments on commit d99d884

Please sign in to comment.