From 03ae313f32cbce7b0948f0248094925a956c61b8 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Thu, 2 Nov 2023 23:09:42 +0100 Subject: [PATCH] fix: simplify the logic --- datafusion/core/src/execution/context/mod.rs | 31 +++++--------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index b39e0176ce9b1..1f1ab446b6268 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -837,16 +837,6 @@ impl SessionContext { .insert(f.name.clone(), Arc::new(f)); } - /// Heuristically determines the format (e.g. parquet, csv) to use with the `table_paths` - fn infer_types(table_paths: &[ListingTableUrl]) -> Option { - let extension = table_paths[0] - .prefix() - .filename() - .map(|filename| filename.split('.').skip(1).collect::>().join(".")) - .unwrap_or("".to_owned()); - Some(extension) - } - /// Creates a [`DataFrame`] for reading a data source. /// /// For more control such as reading multiple files, you can use @@ -866,19 +856,14 @@ impl SessionContext { return exec_err!("No table paths were provided"); } - let extension = Self::infer_types(&table_paths).unwrap(); - // some the file extension might be started with "." and some not - let extension_alternative = ".".to_string() + extension.as_str(); - - if option_extension != extension - && option_extension != extension_alternative - && !extension.is_empty() - { - return exec_err!( - "File extension '{}' does not match the expected extension '{}'", - extension, - option_extension - ); + // check if the file extension matches the expected extension + for path in &table_paths { + if !path.as_str().ends_with(&option_extension) { + let file_name = path.prefix().filename().unwrap_or_default(); + return exec_err!( + "File '{file_name}' does not match the expected extension '{option_extension}'" + ); + } } let resolved_schema = options