Skip to content

Commit

Permalink
fix: simplify the logic
Browse files Browse the repository at this point in the history
  • Loading branch information
Weijun-H committed Nov 2, 2023
1 parent 7c971ea commit 03ae313
Showing 1 changed file with 8 additions and 23 deletions.
31 changes: 8 additions & 23 deletions datafusion/core/src/execution/context/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -837,16 +837,6 @@ impl SessionContext {
.insert(f.name.clone(), Arc::new(f));
}

/// Heuristically determines the format (e.g. parquet, csv) to use with the `table_paths`
fn infer_types(table_paths: &[ListingTableUrl]) -> Option<String> {
let extension = table_paths[0]
.prefix()
.filename()
.map(|filename| filename.split('.').skip(1).collect::<Vec<&str>>().join("."))
.unwrap_or("".to_owned());
Some(extension)
}

/// Creates a [`DataFrame`] for reading a data source.
///
/// For more control such as reading multiple files, you can use
Expand All @@ -866,19 +856,14 @@ impl SessionContext {
return exec_err!("No table paths were provided");
}

let extension = Self::infer_types(&table_paths).unwrap();
// some the file extension might be started with "." and some not
let extension_alternative = ".".to_string() + extension.as_str();

if option_extension != extension
&& option_extension != extension_alternative
&& !extension.is_empty()
{
return exec_err!(
"File extension '{}' does not match the expected extension '{}'",
extension,
option_extension
);
// check if the file extension matches the expected extension
for path in &table_paths {
if !path.as_str().ends_with(&option_extension) {
let file_name = path.prefix().filename().unwrap_or_default();
return exec_err!(
"File '{file_name}' does not match the expected extension '{option_extension}'"
);
}
}

let resolved_schema = options
Expand Down

0 comments on commit 03ae313

Please sign in to comment.