Skip to content

hack out parqeut feature from datafusion-cli #14666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions datafusion-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,12 @@ async-trait = { workspace = true }
aws-config = "1.5.16"
aws-credential-types = "1.2.0"
clap = { version = "4.5.29", features = ["derive", "cargo"] }
datafusion = { workspace = true, features = [
datafusion = { workspace = true, default-features = false, features = [
"avro",
"crypto_expressions",
"datetime_expressions",
"encoding_expressions",
"nested_expressions",
"parquet",
"recursive_protection",
"regex_expressions",
"unicode_expressions",
Expand All @@ -51,7 +50,7 @@ futures = { workspace = true }
mimalloc = { version = "0.1", default-features = false }
object_store = { workspace = true, features = ["aws", "gcp", "http"] }
parking_lot = { workspace = true }
parquet = { workspace = true, default-features = false }
#parquet = { workspace = true, default-features = false }
regex = { workspace = true }
rustyline = "15.0"
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }
Expand Down
8 changes: 6 additions & 2 deletions datafusion-cli/src/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

use crate::cli_context::CliSessionContext;
use crate::exec::{exec_and_print, exec_from_lines};
use crate::functions::{display_all_functions, Function};
//use crate::functions::{display_all_functions, Function};
use crate::print_format::PrintFormat;
use crate::print_options::PrintOptions;
use clap::ValueEnum;
Expand Down Expand Up @@ -102,7 +102,9 @@ impl Command {
Ok(())
}
Self::Quit => exec_err!("Unexpected quit, this should be handled outside"),
Self::ListFunctions => display_all_functions(),
Self::ListFunctions => unimplemented!(),
Self::SearchFunctions(function) => unimplemented!(),
/*
Self::SearchFunctions(function) => {
if let Ok(func) = function.parse::<Function>() {
let details = func.function_details()?;
Expand All @@ -112,6 +114,8 @@ impl Command {
exec_err!("{function} is not a supported function")
}
}

*/
Self::OutputFormat(_) => exec_err!(
"Unexpected change output format, this should be handled outside"
),
Expand Down
2 changes: 1 addition & 1 deletion datafusion-cli/src/exec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ fn config_file_type_from_str(ext: &str) -> Option<ConfigFileType> {
match ext.to_lowercase().as_str() {
"csv" => Some(ConfigFileType::CSV),
"json" => Some(ConfigFileType::JSON),
"parquet" => Some(ConfigFileType::PARQUET),
//"parquet" => Some(ConfigFileType::PARQUET),
_ => None,
}
}
Expand Down
5 changes: 4 additions & 1 deletion datafusion-cli/src/functions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ fn fixed_len_byte_array_to_string(val: Option<&FixedLenByteArray>) -> Option<Str
.unwrap_or_else(|_e| v.to_string())
})
}

/*
#[derive(Debug)]
pub struct ParquetMetadataFunc {}

Expand Down Expand Up @@ -460,3 +460,6 @@ impl TableFunctionImpl for ParquetMetadataFunc {
Ok(Arc::new(parquet_metadata))
}
}


*/
2 changes: 1 addition & 1 deletion datafusion-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ pub mod catalog;
pub mod cli_context;
pub mod command;
pub mod exec;
pub mod functions;
//pub mod functions;
pub mod helper;
pub mod highlighter;
pub mod object_storage;
Expand Down
4 changes: 2 additions & 2 deletions datafusion-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, Memory
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::prelude::SessionContext;
use datafusion_cli::catalog::DynamicObjectStoreCatalog;
use datafusion_cli::functions::ParquetMetadataFunc;
//use datafusion_cli::functions::ParquetMetadataFunc;
use datafusion_cli::{
exec,
pool_type::PoolType,
Expand Down Expand Up @@ -179,7 +179,7 @@ async fn main_inner() -> Result<()> {
ctx.state_weak_ref(),
)));
// register `parquet_metadata` table function to get metadata from parquet files
ctx.register_udtf("parquet_metadata", Arc::new(ParquetMetadataFunc {}));
//ctx.register_udtf("parquet_metadata", Arc::new(ParquetMetadataFunc {}));

let mut print_options = PrintOptions {
format: args.format,
Expand Down
Loading