From 14a9b7e94c4c19fd54c12ddc2722d5866e2af2e9 Mon Sep 17 00:00:00 2001 From: Scott Donnelly Date: Tue, 23 Jul 2024 23:34:09 +0100 Subject: [PATCH 1/2] fix: ensure that RestCatalog passes user config to FileIO --- crates/catalog/rest/src/catalog.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 83cc0ec84..43608a208 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -504,8 +504,15 @@ impl Catalog for RestCatalog { .query::(request) .await?; + let config = resp + .config + .unwrap_or_default() + .into_iter() + .chain(self.user_config.props.clone().into_iter()) + .collect(); + let file_io = self - .load_file_io(resp.metadata_location.as_deref(), resp.config) + .load_file_io(resp.metadata_location.as_deref(), Some(config)) .await?; let table = Table::builder() @@ -542,8 +549,15 @@ impl Catalog for RestCatalog { .query::(request) .await?; + let config = resp + .config + .unwrap_or_default() + .into_iter() + .chain(self.user_config.props.clone().into_iter()) + .collect(); + let file_io = self - .load_file_io(resp.metadata_location.as_deref(), resp.config) + .load_file_io(resp.metadata_location.as_deref(), Some(config)) .await?; let table_builder = Table::builder() From 23fced04f42ce1847cb71dd24d8b7b7b866d7e43 Mon Sep 17 00:00:00 2001 From: Scott Donnelly Date: Mon, 19 Aug 2024 19:41:16 +0100 Subject: [PATCH 2/2] docs: added some doc comments to clarify override order for config --- crates/catalog/rest/src/catalog.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 43608a208..7ceda25e9 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -466,6 +466,11 @@ impl Catalog for RestCatalog { } /// Create a new table inside the namespace. + /// + /// In the resulting table, if there are any config properties that + /// are present in both the response from the REST server and the + /// config provided when creating this `RestCatalog` instance then + /// the value provided locally to the `RestCatalog` will take precedence. async fn create_table( &self, namespace: &NamespaceIdent, @@ -531,6 +536,11 @@ impl Catalog for RestCatalog { } /// Load table from the catalog. + /// + /// If there are any config properties that are present in + /// both the response from the REST server and the config provided + /// when creating this `RestCatalog` instance then the value + /// provided locally to the `RestCatalog` will take precedence. async fn load_table(&self, table: &TableIdent) -> Result { let request = self .context()