Skip to content

Commit 38a8e4e

Browse files
alambalamb
committed
Update for new sqlparser API
1 parent 76f8c0a commit 38a8e4e

File tree

15 files changed

+172
-59
lines changed

15 files changed

+172
-59
lines changed

‎datafusion/common/src/utils/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -775,10 +775,10 @@ pub fn get_available_parallelism() -> usize {
775775

776776
#[cfg(test)]
777777
mod tests {
778+
use super::*;
778779
use crate::ScalarValue::Null;
779780
use arrow::array::Float64Array;
780-
781-
use super::*;
781+
use sqlparser::tokenizer::Span;
782782

783783
#[test]
784784
fn test_bisect_linear_left_and_right() -> Result<()> {
@@ -1006,6 +1006,7 @@ mod tests {
10061006
let expected_parsed = vec![Ident {
10071007
value: identifier.to_string(),
10081008
quote_style,
1009+
span: Span::empty(),
10091010
}];
10101011

10111012
assert_eq!(

‎datafusion/core/tests/user_defined/user_defined_scalar_functions.rs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,6 @@ use arrow_array::{
2727
Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,
2828
};
2929
use arrow_schema::{DataType, Field, Schema};
30-
use parking_lot::Mutex;
31-
use regex::Regex;
32-
use sqlparser::ast::Ident;
33-
3430
use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState};
3531
use datafusion::prelude::*;
3632
use datafusion::{execution::registry::FunctionRegistry, test_util};
@@ -48,6 +44,10 @@ use datafusion_expr::{
4844
Volatility,
4945
};
5046
use datafusion_functions_nested::range::range_udf;
47+
use parking_lot::Mutex;
48+
use regex::Regex;
49+
use sqlparser::ast::Ident;
50+
use sqlparser::tokenizer::Span;
5151

5252
/// test that casting happens on udfs.
5353
/// c11 is f32, but `custom_sqrt` requires f64. Casting happens but the logical plan and
@@ -1187,6 +1187,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
11871187
name: Some(Ident {
11881188
value: "name".into(),
11891189
quote_style: None,
1190+
span: Span::empty(),
11901191
}),
11911192
data_type: DataType::Utf8,
11921193
default_expr: None,
@@ -1196,6 +1197,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
11961197
language: Some(Ident {
11971198
value: "plrust".into(),
11981199
quote_style: None,
1200+
span: Span::empty(),
11991201
}),
12001202
behavior: None,
12011203
function_body: Some(lit(body)),

‎datafusion/expr/src/expr.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,8 @@ use sqlparser::ast::{
222222
/// // to 42 = 5 AND b = 6
223223
/// assert_eq!(rewritten.data, lit(42).eq(lit(5)).and(col("b").eq(lit(6))));
224224
#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
225+
// TODO make the enum smaller with more boxing (looks like Wildcard is now bigger)
226+
#[allow(clippy::large_enum_variant)]
225227
pub enum Expr {
226228
/// An expression with a specific name.
227229
Alias(Alias),

‎datafusion/sql/src/expr/function.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,11 @@ impl FunctionArgs {
169169
"Calling {name}: SEPARATOR not supported in function arguments: {sep}"
170170
)
171171
}
172+
FunctionArgumentClause::JsonNullClause(jn) => {
173+
return not_impl_err!(
174+
"Calling {name}: JSON NULL clause not supported in function arguments: {jn}"
175+
)
176+
}
172177
}
173178
}
174179

‎datafusion/sql/src/expr/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -565,11 +565,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
565565
}
566566
not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}")
567567
}
568-
SQLExpr::Wildcard => Ok(Expr::Wildcard {
568+
SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {
569569
qualifier: None,
570570
options: WildcardOptions::default(),
571571
}),
572-
SQLExpr::QualifiedWildcard(object_name) => Ok(Expr::Wildcard {
572+
SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard {
573573
qualifier: Some(self.object_name_to_table_reference(object_name)?),
574574
options: WildcardOptions::default(),
575575
}),

‎datafusion/sql/src/parser.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,7 @@ mod tests {
875875
use super::*;
876876
use sqlparser::ast::Expr::Identifier;
877877
use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident};
878+
use sqlparser::tokenizer::Span;
878879

879880
fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> {
880881
let statements = DFParser::parse_sql(sql)?;
@@ -910,6 +911,7 @@ mod tests {
910911
name: Ident {
911912
value: name.into(),
912913
quote_style: None,
914+
span: Span::empty(),
913915
},
914916
data_type,
915917
collation: None,
@@ -1218,6 +1220,7 @@ mod tests {
12181220
expr: Identifier(Ident {
12191221
value: "c1".to_owned(),
12201222
quote_style: None,
1223+
span: Span::empty(),
12211224
}),
12221225
asc,
12231226
nulls_first,
@@ -1249,6 +1252,7 @@ mod tests {
12491252
expr: Identifier(Ident {
12501253
value: "c1".to_owned(),
12511254
quote_style: None,
1255+
span: Span::empty(),
12521256
}),
12531257
asc: Some(true),
12541258
nulls_first: None,
@@ -1258,6 +1262,7 @@ mod tests {
12581262
expr: Identifier(Ident {
12591263
value: "c2".to_owned(),
12601264
quote_style: None,
1265+
span: Span::empty(),
12611266
}),
12621267
asc: Some(false),
12631268
nulls_first: Some(true),
@@ -1289,11 +1294,13 @@ mod tests {
12891294
left: Box::new(Identifier(Ident {
12901295
value: "c1".to_owned(),
12911296
quote_style: None,
1297+
span: Span::empty(),
12921298
})),
12931299
op: BinaryOperator::Minus,
12941300
right: Box::new(Identifier(Ident {
12951301
value: "c2".to_owned(),
12961302
quote_style: None,
1303+
span: Span::empty(),
12971304
})),
12981305
},
12991306
asc: Some(true),
@@ -1334,11 +1341,13 @@ mod tests {
13341341
left: Box::new(Identifier(Ident {
13351342
value: "c1".to_owned(),
13361343
quote_style: None,
1344+
span: Span::empty(),
13371345
})),
13381346
op: BinaryOperator::Minus,
13391347
right: Box::new(Identifier(Ident {
13401348
value: "c2".to_owned(),
13411349
quote_style: None,
1350+
span: Span::empty(),
13421351
})),
13431352
},
13441353
asc: Some(true),

‎datafusion/sql/src/planner.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
339339
plan: LogicalPlan,
340340
alias: TableAlias,
341341
) -> Result<LogicalPlan> {
342-
let plan = self.apply_expr_alias(plan, alias.columns)?;
342+
let idents = alias.columns.into_iter().map(|c| c.name).collect();
343+
let plan = self.apply_expr_alias(plan, idents)?;
343344

344345
LogicalPlanBuilder::from(plan)
345346
.alias(TableReference::bare(

‎datafusion/sql/src/select.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -654,6 +654,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
654654
opt_rename,
655655
opt_replace: _opt_replace,
656656
opt_ilike: _opt_ilike,
657+
wildcard_token: _wildcard_token,
657658
} = options;
658659

659660
if opt_rename.is_some() {

‎datafusion/sql/src/statement.rs

Lines changed: 106 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ use datafusion_expr::{
5454
TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart,
5555
Volatility, WriteOp,
5656
};
57-
use sqlparser::ast::{self, SqliteOnConflict};
57+
use sqlparser::ast::{self, ShowStatementIn, ShowStatementOptions, SqliteOnConflict};
5858
use sqlparser::ast::{
5959
Assignment, AssignmentTarget, ColumnDef, CreateIndex, CreateTable,
6060
CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, FromTable, Ident, Insert,
6161
ObjectName, ObjectType, OneOrManyWithParens, Query, SchemaName, SetExpr,
62-
ShowCreateObject, ShowStatementFilter, Statement, TableConstraint, TableFactor,
63-
TableWithJoins, TransactionMode, UnaryOperator, Value,
62+
ShowCreateObject, Statement, TableConstraint, TableFactor, TableWithJoins,
63+
TransactionMode, UnaryOperator, Value,
6464
};
6565
use sqlparser::parser::ParserError::ParserError;
6666

@@ -685,19 +685,99 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
685685
Statement::ShowTables {
686686
extended,
687687
full,
688-
db_name,
689-
filter,
690-
// SHOW TABLES IN/FROM are equivalent, this field specifies which the user
691-
// specified, but it doesn't affect the plan so ignore the field
692-
clause: _,
693-
} => self.show_tables_to_plan(extended, full, db_name, filter),
688+
terse,
689+
history,
690+
external,
691+
show_options,
692+
} => {
693+
// We only support the basic "SHOW TABLES"
694+
// https://github.com/apache/datafusion/issues/3188
695+
if extended {
696+
return not_impl_err!("SHOW TABLES EXTENDED not supported")?;
697+
}
698+
if full {
699+
return not_impl_err!("SHOW FULL TABLES not supported")?;
700+
}
701+
if terse {
702+
return not_impl_err!("SHOW TERSE TABLES not supported")?;
703+
}
704+
if history {
705+
return not_impl_err!("SHOW TABLES HISTORY not supported")?;
706+
}
707+
if external {
708+
return not_impl_err!("SHOW EXTERNAL TABLES not supported")?;
709+
}
710+
let ShowStatementOptions {
711+
show_in,
712+
starts_with,
713+
limit,
714+
limit_from,
715+
filter_position,
716+
} = show_options;
717+
if show_in.is_some() {
718+
return not_impl_err!("SHOW TABLES IN not supported")?;
719+
}
720+
if starts_with.is_some() {
721+
return not_impl_err!("SHOW TABLES LIKE not supported")?;
722+
}
723+
if limit.is_some() {
724+
return not_impl_err!("SHOW TABLES LIMIT not supported")?;
725+
}
726+
if limit_from.is_some() {
727+
return not_impl_err!("SHOW TABLES LIMIT FROM not supported")?;
728+
}
729+
if filter_position.is_some() {
730+
return not_impl_err!("SHOW TABLES FILTER not supported")?;
731+
}
732+
self.show_tables_to_plan()
733+
}
694734

695735
Statement::ShowColumns {
696736
extended,
697737
full,
698-
table_name,
699-
filter,
700-
} => self.show_columns_to_plan(extended, full, table_name, filter),
738+
show_options,
739+
} => {
740+
let ShowStatementOptions {
741+
show_in,
742+
starts_with,
743+
limit,
744+
limit_from,
745+
filter_position,
746+
} = show_options;
747+
if starts_with.is_some() {
748+
return not_impl_err!("SHOW COLUMNS LIKE not supported")?;
749+
}
750+
if limit.is_some() {
751+
return not_impl_err!("SHOW COLUMNS LIMIT not supported")?;
752+
}
753+
if limit_from.is_some() {
754+
return not_impl_err!("SHOW COLUMNS LIMIT FROM not supported")?;
755+
}
756+
if filter_position.is_some() {
757+
return not_impl_err!(
758+
"SHOW COLUMNS with WHERE or LIKE is not supported"
759+
)?;
760+
}
761+
let Some(ShowStatementIn {
762+
// specifies if the syntax was `SHOW COLUMNS IN` or `SHOW
763+
// COLUMNS FROM` which is not different in DataFusion
764+
clause: _,
765+
parent_type,
766+
parent_name,
767+
}) = show_in
768+
else {
769+
return plan_err!("SHOW COLUMNS requires a table name");
770+
};
771+
772+
if let Some(parent_type) = parent_type {
773+
return not_impl_err!("SHOW COLUMNS IN {parent_type} not supported");
774+
}
775+
let Some(table_name) = parent_name else {
776+
return plan_err!("SHOW COLUMNS requires a table name");
777+
};
778+
779+
self.show_columns_to_plan(extended, full, table_name)
780+
}
701781

702782
Statement::Insert(Insert {
703783
or,
@@ -766,10 +846,14 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
766846
from,
767847
selection,
768848
returning,
849+
or,
769850
} => {
770851
if returning.is_some() {
771852
plan_err!("Update-returning clause not yet supported")?;
772853
}
854+
if or.is_some() {
855+
plan_err!("ON conflict not supported")?;
856+
}
773857
self.update_to_plan(table, assignments, from, selection)
774858
}
775859

@@ -1065,24 +1149,12 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
10651149
}
10661150

10671151
/// Generate a logical plan from a "SHOW TABLES" query
1068-
fn show_tables_to_plan(
1069-
&self,
1070-
extended: bool,
1071-
full: bool,
1072-
db_name: Option<Ident>,
1073-
filter: Option<ShowStatementFilter>,
1074-
) -> Result<LogicalPlan> {
1152+
fn show_tables_to_plan(&self) -> Result<LogicalPlan> {
10751153
if self.has_table("information_schema", "tables") {
1076-
// We only support the basic "SHOW TABLES"
1077-
// https://github.com/apache/datafusion/issues/3188
1078-
if db_name.is_some() || filter.is_some() || full || extended {
1079-
plan_err!("Unsupported parameters to SHOW TABLES")
1080-
} else {
1081-
let query = "SELECT * FROM information_schema.tables;";
1082-
let mut rewrite = DFParser::parse_sql(query)?;
1083-
assert_eq!(rewrite.len(), 1);
1084-
self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1
1085-
}
1154+
let query = "SELECT * FROM information_schema.tables;";
1155+
let mut rewrite = DFParser::parse_sql(query)?;
1156+
assert_eq!(rewrite.len(), 1);
1157+
self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1
10861158
} else {
10871159
plan_err!("SHOW TABLES is not supported unless information_schema is enabled")
10881160
}
@@ -1842,22 +1914,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
18421914
extended: bool,
18431915
full: bool,
18441916
sql_table_name: ObjectName,
1845-
filter: Option<ShowStatementFilter>,
18461917
) -> Result<LogicalPlan> {
1847-
if filter.is_some() {
1848-
return plan_err!("SHOW COLUMNS with WHERE or LIKE is not supported");
1849-
}
1918+
// Figure out the where clause
1919+
let where_clause = object_name_to_qualifier(
1920+
&sql_table_name,
1921+
self.options.enable_ident_normalization,
1922+
);
18501923

18511924
if !self.has_table("information_schema", "columns") {
18521925
return plan_err!(
18531926
"SHOW COLUMNS is not supported unless information_schema is enabled"
18541927
);
18551928
}
1856-
// Figure out the where clause
1857-
let where_clause = object_name_to_qualifier(
1858-
&sql_table_name,
1859-
self.options.enable_ident_normalization,
1860-
);
18611929

18621930
// Do a table lookup to verify the table exists
18631931
let table_ref = self.object_name_to_table_reference(sql_table_name)?;

0 commit comments

Comments
 (0)