From 66d28979b74701b8acbbdeb246c4156054981318 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sat, 15 Mar 2025 11:49:03 +0800 Subject: [PATCH 01/11] Support logic optimize rule to pass the case that Utf8view datatype combined with Utf8 datatype --- datafusion/expr/src/logical_plan/invariants.rs | 6 ++++-- datafusion/sqllogictest/test_files/expr.slt | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs index f4ac33b6afef..65c01294d766 100644 --- a/datafusion/expr/src/logical_plan/invariants.rs +++ b/datafusion/expr/src/logical_plan/invariants.rs @@ -112,9 +112,11 @@ fn assert_valid_semantic_plan(plan: &LogicalPlan) -> Result<()> { /// Returns an error if the plan does not have the expected schema. /// Ignores metadata and nullability. pub fn assert_expected_schema(schema: &DFSchemaRef, plan: &LogicalPlan) -> Result<()> { - let equivalent = plan.schema().equivalent_names_and_types(schema); + let compatible = plan + .schema() + .check_arrow_schema_type_compatible(&schema.as_ref().into()); - if !equivalent { + if compatible.is_err() { internal_err!( "Failed due to a difference in schemas, original schema: {:?}, new schema: {:?}", schema, diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 74e9fe065a73..24f7c3ea15c6 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -478,8 +478,14 @@ a statement ok create table foo (a varchar, b varchar) as values ('a', 'b'); + +query T +SELECT concat_ws('', a, b,'c') from foo +---- +abc + query T -SELECT concat_ws('',a,b,'c') from foo +SELECT concat_ws('',arrow_cast(a, 'Utf8View'),arrow_cast(b, 'Utf8View'),'c') from foo ---- abc From 36df94ed23117ad476f1401edc686cc143ca36dc Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sat, 15 Mar 2025 11:49:08 +0800 Subject: [PATCH 02/11] Support logic optimize rule to pass the case that Utf8view datatype combined with Utf8 datatype --- datafusion/optimizer/src/optimizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 3a69bd91e749..4fc334d4cc8e 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -447,7 +447,7 @@ fn assert_valid_optimization( plan: &LogicalPlan, prev_schema: &Arc, ) -> Result<()> { - // verify invariant: optimizer passes should not change the schema + // verify invariant: optimizer passes should not change the schema if the schema can't be cast from the previous schema. // Refer to assert_expected_schema(prev_schema, plan)?; From 308307b5668d275e8565a3d0efc7951dff34b24f Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 12:04:48 +0800 Subject: [PATCH 03/11] fix test --- datafusion/common/src/dfschema.rs | 3 ++- datafusion/expr/src/logical_plan/invariants.rs | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 65bb40810f18..56547c551023 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -622,7 +622,8 @@ impl DFSchema { self_fields.zip(other_fields).all(|((q1, f1), (q2, f2))| { q1 == q2 && f1.name() == f2.name() - && Self::datatype_is_semantically_equal(f1.data_type(), f2.data_type()) + && (Self::datatype_is_semantically_equal(f1.data_type(), f2.data_type()) + || !can_cast_types(f2.data_type(), f1.data_type())) }) } diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs index 65c01294d766..0645c3c2670d 100644 --- a/datafusion/expr/src/logical_plan/invariants.rs +++ b/datafusion/expr/src/logical_plan/invariants.rs @@ -112,11 +112,8 @@ fn assert_valid_semantic_plan(plan: &LogicalPlan) -> Result<()> { /// Returns an error if the plan does not have the expected schema. /// Ignores metadata and nullability. pub fn assert_expected_schema(schema: &DFSchemaRef, plan: &LogicalPlan) -> Result<()> { - let compatible = plan - .schema() - .check_arrow_schema_type_compatible(&schema.as_ref().into()); - - if compatible.is_err() { + let equivalent = plan.schema().equivalent_names_and_types(schema); + if !equivalent { internal_err!( "Failed due to a difference in schemas, original schema: {:?}, new schema: {:?}", schema, From c7c0c98d7e995512d1d1b29a6d30668f23ece0ef Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 12:09:02 +0800 Subject: [PATCH 04/11] fix --- datafusion/common/src/dfschema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 56547c551023..217fc88c5153 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -623,7 +623,7 @@ impl DFSchema { q1 == q2 && f1.name() == f2.name() && (Self::datatype_is_semantically_equal(f1.data_type(), f2.data_type()) - || !can_cast_types(f2.data_type(), f1.data_type())) + || can_cast_types(f2.data_type(), f1.data_type())) }) } From c75cd88cf6e8fe191351f22c170ff7f2906a4a54 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 12:09:39 +0800 Subject: [PATCH 05/11] fix --- datafusion/common/src/dfschema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 217fc88c5153..4df2caff1b3d 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -605,7 +605,7 @@ impl DFSchema { } /// Returns true if the two schemas have the same qualified named - /// fields with the same data types. Returns false otherwise. + /// fields with the compatible data types. Returns false otherwise. /// /// This is a specialized version of Eq that ignores differences /// in nullability and metadata. From e1e57a865aec85d1d507fd9a88bf252a47e1e687 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 12:12:48 +0800 Subject: [PATCH 06/11] fmt --- datafusion/common/src/dfschema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 4df2caff1b3d..b602b464c8b6 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -623,7 +623,7 @@ impl DFSchema { q1 == q2 && f1.name() == f2.name() && (Self::datatype_is_semantically_equal(f1.data_type(), f2.data_type()) - || can_cast_types(f2.data_type(), f1.data_type())) + || can_cast_types(f2.data_type(), f1.data_type())) }) } From 9b8a0112187782394fba58c1d1dc627fe5c52b79 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 12:37:46 +0800 Subject: [PATCH 07/11] clean code --- datafusion/common/src/dfschema.rs | 66 +++++++++---------- .../expr/src/logical_plan/invariants.rs | 7 +- datafusion/optimizer/src/optimizer.rs | 30 ++++----- .../substrait/src/logical_plan/consumer.rs | 2 +- 4 files changed, 49 insertions(+), 56 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index b602b464c8b6..92a7a81172be 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -563,29 +563,6 @@ impl DFSchema { .all(|(dffield, arrowfield)| dffield.name() == arrowfield.name()) } - /// Check to see if fields in 2 Arrow schemas are compatible - pub fn check_arrow_schema_type_compatible( - &self, - arrow_schema: &Schema, - ) -> Result<()> { - let self_arrow_schema: Schema = self.into(); - self_arrow_schema - .fields() - .iter() - .zip(arrow_schema.fields().iter()) - .try_for_each(|(l_field, r_field)| { - if !can_cast_types(r_field.data_type(), l_field.data_type()) { - _plan_err!("Column {} (type: {}) is not compatible with column {} (type: {})", - r_field.name(), - r_field.data_type(), - l_field.name(), - l_field.data_type()) - } else { - Ok(()) - } - }) - } - /// Returns true if the two schemas have the same qualified named /// fields with logically equivalent data types. Returns false otherwise. /// @@ -613,18 +590,41 @@ impl DFSchema { /// Use [DFSchema]::logically_equivalent_names_and_types for a weaker /// logical type checking, which for example would consider a dictionary /// encoded UTF8 array to be equivalent to a plain UTF8 array. - pub fn equivalent_names_and_types(&self, other: &Self) -> bool { + pub fn equivalent_names_and_types(&self, other: &Self) -> Result<()> { + // case 1 : schema length mismatch if self.fields().len() != other.fields().len() { - return false; + _plan_err!( + "Schema mismatch: the schema length are not same \ + Expected schema length: {}, got: {}", + self.fields().len(), + other.fields().len() + ) + } else { + // case 2 : schema length match, but fields mismatch + // check if the fields name are the same and have the same data types + self.fields() + .iter() + .zip(other.fields().iter()) + .try_for_each(|(f1, f2)| { + if f1.name() != f2.name() + || (!DFSchema::datatype_is_semantically_equal( + f1.data_type(), + f2.data_type(), + ) && !can_cast_types(f2.data_type(), f1.data_type())) + { + _plan_err!( + "Schema mismatch: Expected field '{}' with type {:?}, \ + but got '{}' with type {:?}.", + f1.name(), + f1.data_type(), + f2.name(), + f2.data_type() + ) + } else { + Ok(()) + } + }) } - let self_fields = self.iter(); - let other_fields = other.iter(); - self_fields.zip(other_fields).all(|((q1, f1), (q2, f2))| { - q1 == q2 - && f1.name() == f2.name() - && (Self::datatype_is_semantically_equal(f1.data_type(), f2.data_type()) - || can_cast_types(f2.data_type(), f1.data_type())) - }) } /// Checks if two [`DataType`]s are logically equal. This is a notably weaker constraint diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs index 0645c3c2670d..75c4eb1dd0c7 100644 --- a/datafusion/expr/src/logical_plan/invariants.rs +++ b/datafusion/expr/src/logical_plan/invariants.rs @@ -112,10 +112,11 @@ fn assert_valid_semantic_plan(plan: &LogicalPlan) -> Result<()> { /// Returns an error if the plan does not have the expected schema. /// Ignores metadata and nullability. pub fn assert_expected_schema(schema: &DFSchemaRef, plan: &LogicalPlan) -> Result<()> { - let equivalent = plan.schema().equivalent_names_and_types(schema); - if !equivalent { + let compatible = plan.schema().equivalent_names_and_types(schema); + + if let Err(e) = compatible { internal_err!( - "Failed due to a difference in schemas, original schema: {:?}, new schema: {:?}", + "Failed due to a difference in schemas: {e}, original schema: {:?}, new schema: {:?}", schema, plan.schema() ) diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 4fc334d4cc8e..58a7d1ebefed 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -507,25 +507,17 @@ mod tests { let err = opt.optimize(plan, &config, &observe).unwrap_err(); assert!(err.strip_backtrace().starts_with( "Optimizer rule 'get table_scan rule' failed\n\ - caused by\n\ - Check optimizer-specific invariants after optimizer rule: get table_scan rule\n\ - caused by\n\ - Internal error: Failed due to a difference in schemas, \ - original schema: DFSchema { inner: Schema { \ - fields: [], \ - metadata: {} }, \ - field_qualifiers: [], \ - functional_dependencies: FunctionalDependencies { deps: [] } \ - }, \ - new schema: DFSchema { inner: Schema { \ - fields: [\ - Field { name: \"a\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ - Field { name: \"b\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ - Field { name: \"c\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }\ - ], \ - metadata: {} }, \ - field_qualifiers: [Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" })], \ - functional_dependencies: FunctionalDependencies { deps: [] } }", + caused by\n\ + Check optimizer-specific invariants after optimizer rule: get table_scan rule\n\ + caused by\n\ + Internal error: Failed due to a difference in schemas: Error during planning: Schema mismatch: \ + the schema length are not same Expected schema length: 3, got: 0, original schema: DFSchema \ + { inner: Schema { fields: [], metadata: {} }, field_qualifiers: [], functional_dependencies: FunctionalDependencies { deps: [] } }, new schema: DFSchema { inner: Schema { fields: [\ + Field { name: \"a\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ + Field { name: \"b\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ + Field { name: \"c\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }\ + ], metadata: {} }, field_qualifiers: [Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" }), \ + Some(Bare { table: \"test\" })], functional_dependencies: FunctionalDependencies { deps: [] } }." )); } diff --git a/datafusion/substrait/src/logical_plan/consumer.rs b/datafusion/substrait/src/logical_plan/consumer.rs index 24eb23ded5a7..8de4840b610d 100644 --- a/datafusion/substrait/src/logical_plan/consumer.rs +++ b/datafusion/substrait/src/logical_plan/consumer.rs @@ -776,7 +776,7 @@ pub async fn from_substrait_plan_with_consumer( return Ok(plan); } let renamed_schema = make_renamed_schema(plan.schema(), &root.names)?; - if renamed_schema.equivalent_names_and_types(plan.schema()) { + if renamed_schema.equivalent_names_and_types(plan.schema()).is_ok() { // Nothing to do if the schema is already equivalent return Ok(plan); } From 4187e9fddaa80923603769f967473566906d27d3 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 13:58:55 +0800 Subject: [PATCH 08/11] fix test --- datafusion/optimizer/src/optimizer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 58a7d1ebefed..2a842e010b84 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -511,13 +511,13 @@ mod tests { Check optimizer-specific invariants after optimizer rule: get table_scan rule\n\ caused by\n\ Internal error: Failed due to a difference in schemas: Error during planning: Schema mismatch: \ - the schema length are not same Expected schema length: 3, got: 0, original schema: DFSchema \ - { inner: Schema { fields: [], metadata: {} }, field_qualifiers: [], functional_dependencies: FunctionalDependencies { deps: [] } }, new schema: DFSchema { inner: Schema { fields: [\ + the schema length are not same Expected schema length: 3, got: 0, original schema: DFSchema { inner: Schema { fields: [], metadata: {} }, \ + field_qualifiers: [], functional_dependencies: FunctionalDependencies { deps: [] } }, new schema: DFSchema { inner: Schema { fields: [\ Field { name: \"a\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ Field { name: \"b\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ Field { name: \"c\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }\ - ], metadata: {} }, field_qualifiers: [Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" }), \ - Some(Bare { table: \"test\" })], functional_dependencies: FunctionalDependencies { deps: [] } }." + ], metadata: {} }, field_qualifiers: [Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" })], \ + functional_dependencies: FunctionalDependencies { deps: [] } }." )); } From 3ef7b29a31e9b70ba7c1741aac84add8939244e3 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Sun, 16 Mar 2025 14:28:13 +0800 Subject: [PATCH 09/11] Fix test --- datafusion/optimizer/src/optimizer.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 2a842e010b84..04d73fe3ab87 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -459,7 +459,9 @@ mod tests { use std::sync::{Arc, Mutex}; use datafusion_common::tree_node::Transformed; - use datafusion_common::{plan_err, DFSchema, DFSchemaRef, DataFusionError, Result}; + use datafusion_common::{ + assert_contains, plan_err, DFSchema, DFSchemaRef, DataFusionError, Result, + }; use datafusion_expr::logical_plan::EmptyRelation; use datafusion_expr::{col, lit, LogicalPlan, LogicalPlanBuilder, Projection}; @@ -505,20 +507,9 @@ mod tests { schema: Arc::new(DFSchema::empty()), }); let err = opt.optimize(plan, &config, &observe).unwrap_err(); - assert!(err.strip_backtrace().starts_with( - "Optimizer rule 'get table_scan rule' failed\n\ - caused by\n\ - Check optimizer-specific invariants after optimizer rule: get table_scan rule\n\ - caused by\n\ - Internal error: Failed due to a difference in schemas: Error during planning: Schema mismatch: \ - the schema length are not same Expected schema length: 3, got: 0, original schema: DFSchema { inner: Schema { fields: [], metadata: {} }, \ - field_qualifiers: [], functional_dependencies: FunctionalDependencies { deps: [] } }, new schema: DFSchema { inner: Schema { fields: [\ - Field { name: \"a\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ - Field { name: \"b\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, \ - Field { name: \"c\", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }\ - ], metadata: {} }, field_qualifiers: [Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" }), Some(Bare { table: \"test\" })], \ - functional_dependencies: FunctionalDependencies { deps: [] } }." - )); + + // Simplify assert to check the error message contains the expected message, which is only the schema length mismatch + assert_contains!(err.strip_backtrace(), "Schema mismatch: the schema length are not same Expected schema length: 3, got: 0"); } #[test] From a10f46a57d3cbaef78f8f0b140e081607c7f9380 Mon Sep 17 00:00:00 2001 From: zhuqi-lucas <821684824@qq.com> Date: Tue, 18 Mar 2025 12:45:20 +0800 Subject: [PATCH 10/11] address comments --- datafusion/common/src/dfschema.rs | 31 ++++++++++++++++++- .../expr/src/logical_plan/invariants.rs | 2 +- .../substrait/src/logical_plan/consumer.rs | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 92a7a81172be..e1890b0c330a 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -563,6 +563,30 @@ impl DFSchema { .all(|(dffield, arrowfield)| dffield.name() == arrowfield.name()) } + /// Check to see if fields in 2 Arrow schemas are compatible + #[deprecated(since = "47.0.0", note = "This method is no longer used")] + pub fn check_arrow_schema_type_compatible( + &self, + arrow_schema: &Schema, + ) -> Result<()> { + let self_arrow_schema: Schema = self.into(); + self_arrow_schema + .fields() + .iter() + .zip(arrow_schema.fields().iter()) + .try_for_each(|(l_field, r_field)| { + if !can_cast_types(r_field.data_type(), l_field.data_type()) { + _plan_err!("Column {} (type: {}) is not compatible with column {} (type: {})", + r_field.name(), + r_field.data_type(), + l_field.name(), + l_field.data_type()) + } else { + Ok(()) + } + }) + } + /// Returns true if the two schemas have the same qualified named /// fields with logically equivalent data types. Returns false otherwise. /// @@ -590,7 +614,12 @@ impl DFSchema { /// Use [DFSchema]::logically_equivalent_names_and_types for a weaker /// logical type checking, which for example would consider a dictionary /// encoded UTF8 array to be equivalent to a plain UTF8 array. - pub fn equivalent_names_and_types(&self, other: &Self) -> Result<()> { + #[deprecated(since = "47.0.0", note = "Use has_equivalent_names_and_types` instead")] + pub fn equivalent_names_and_types(&self, other: &Self) -> bool { + self.has_equivalent_names_and_types(other).is_ok() + } + + pub fn has_equivalent_names_and_types(&self, other: &Self) -> Result<()> { // case 1 : schema length mismatch if self.fields().len() != other.fields().len() { _plan_err!( diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs index 75c4eb1dd0c7..d83410bf99c9 100644 --- a/datafusion/expr/src/logical_plan/invariants.rs +++ b/datafusion/expr/src/logical_plan/invariants.rs @@ -112,7 +112,7 @@ fn assert_valid_semantic_plan(plan: &LogicalPlan) -> Result<()> { /// Returns an error if the plan does not have the expected schema. /// Ignores metadata and nullability. pub fn assert_expected_schema(schema: &DFSchemaRef, plan: &LogicalPlan) -> Result<()> { - let compatible = plan.schema().equivalent_names_and_types(schema); + let compatible = plan.schema().has_equivalent_names_and_types(schema); if let Err(e) = compatible { internal_err!( diff --git a/datafusion/substrait/src/logical_plan/consumer.rs b/datafusion/substrait/src/logical_plan/consumer.rs index 8de4840b610d..d30eb5145516 100644 --- a/datafusion/substrait/src/logical_plan/consumer.rs +++ b/datafusion/substrait/src/logical_plan/consumer.rs @@ -776,7 +776,7 @@ pub async fn from_substrait_plan_with_consumer( return Ok(plan); } let renamed_schema = make_renamed_schema(plan.schema(), &root.names)?; - if renamed_schema.equivalent_names_and_types(plan.schema()).is_ok() { + if renamed_schema.has_equivalent_names_and_types(plan.schema()).is_ok() { // Nothing to do if the schema is already equivalent return Ok(plan); } From 2fbe7a6ba81ecdb922dc64081f2ac953a5ef0924 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 18 Mar 2025 13:48:45 -0400 Subject: [PATCH 11/11] move docs --- datafusion/common/src/dfschema.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index e1890b0c330a..43d082f9dc93 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -605,20 +605,22 @@ impl DFSchema { }) } - /// Returns true if the two schemas have the same qualified named - /// fields with the compatible data types. Returns false otherwise. - /// - /// This is a specialized version of Eq that ignores differences - /// in nullability and metadata. - /// - /// Use [DFSchema]::logically_equivalent_names_and_types for a weaker - /// logical type checking, which for example would consider a dictionary - /// encoded UTF8 array to be equivalent to a plain UTF8 array. #[deprecated(since = "47.0.0", note = "Use has_equivalent_names_and_types` instead")] pub fn equivalent_names_and_types(&self, other: &Self) -> bool { self.has_equivalent_names_and_types(other).is_ok() } + /// Returns Ok if the two schemas have the same qualified named + /// fields with the compatible data types. + /// + /// Returns an `Err` with a message otherwise. + /// + /// This is a specialized version of Eq that ignores differences in + /// nullability and metadata. + /// + /// Use [DFSchema]::logically_equivalent_names_and_types for a weaker + /// logical type checking, which for example would consider a dictionary + /// encoded UTF8 array to be equivalent to a plain UTF8 array. pub fn has_equivalent_names_and_types(&self, other: &Self) -> Result<()> { // case 1 : schema length mismatch if self.fields().len() != other.fields().len() {