Merge branch 'main' into datafusion-integration-test

Author: gruuya

.github/actions/setup-builder/action.yml

@@ -21,18 +21,24 @@ name: Prepare Rust Builder
 description: 'Prepare Rust Build Environment'
 inputs:
   rust-version:
-    description: 'version of rust to install (e.g. stable)'
-    required: true
-    default: 'stable'
+    description: 'version of rust to install and use'
 runs:
   using: "composite"
   steps:
-    - name: Setup Rust toolchain
+    - name: Setup specified Rust toolchain
       shell: bash
+      if: ${{ inputs.rust-version != '' }}
       run: |
         echo "Installing ${{ inputs.rust-version }}"
         rustup toolchain install ${{ inputs.rust-version }}
-        rustup default ${{ inputs.rust-version }}
+        rustup override set ${{ inputs.rust-version }}
+        rustup component add rustfmt clippy
+    - name: Setup Rust toolchain according to rust-toolchain.toml
+      shell: bash
+      if: ${{ inputs.rust-version == '' }}
+      run: |
+        echo "Installing toolchain according to rust-toolchain.toml"
+        rustup show
         rustup component add rustfmt clippy
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766

.github/dependabot.yml

@@ -30,3 +30,4 @@ updates:
     schedule:
       interval: "weekly"
       day: "sunday"
+    open-pull-requests-limit: 50

.github/workflows/ci.yml

@@ -43,6 +43,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
       - name: Check License Header
         uses: apache/skywalking-eyes/[email protected]
 
@@ -89,12 +92,10 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
 
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
-  
+
       - name: Build
         run: make build
 
@@ -109,6 +110,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
 
@@ -122,8 +126,6 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
 
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
@@ -136,3 +138,24 @@ jobs:
 
       - name: Doc Test
         run: cargo test --no-fail-fast --doc --all-features --workspace
+
+  msrv:
+    name: Verify MSRV
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Nightly Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Generate minimal versions lockfile
+        run: |
+          cargo generate-lockfile -Z direct-minimal-versions -Z minimal-versions
+          # Some dependencies don't correctly specify a minimal version for their dependencies and will fail to build.
+          # So we update these transitive dependencies here.
+          cargo update tap faststr metainfo linkedbytes
+      - name: Setup MSRV Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
+      - name: Check MSRV
+        run: |
+          cargo +${{ env.rust_msrv }} check --locked --workspace --exclude iceberg-datafusion --exclude iceberg-catalog-s3tables --exclude iceberg-integration-tests

.github/workflows/ci_typos.yml

@@ -42,4 +42,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.28.4
+        uses: crate-ci/typos@v1.29.5

Cargo.toml

@@ -41,62 +41,63 @@ rust-version = "1.77.1"
 anyhow = "1.0.72"
 apache-avro = "0.17"
 array-init = "2"
-arrow-arith = { version = "53" }
-arrow-array = { version = "53" }
-arrow-cast = { version = "53" }
-arrow-ord = { version = "53" }
-arrow-schema = { version = "53" }
-arrow-select = { version = "53" }
-arrow-string = { version = "53" }
+arrow-arith = { version = "53.3.0" }
+arrow-array = { version = "53.4.0" }
+arrow-cast = { version = "53.4.0" }
+arrow-ord = { version = "53.4.0" }
+arrow-schema = { version = "53.4.0" }
+arrow-select = { version = "53.4.0" }
+arrow-string = { version = "53.4.0" }
 async-stream = "0.3.5"
-async-trait = "0.1"
+async-trait = "0.1.86"
 async-std = "1.12"
-aws-config = "1.1.8"
-aws-sdk-glue = "1.21"
+aws-config = "1"
+aws-sdk-glue = "1.39"
 bimap = "0.6"
 bitvec = "1.0.1"
-bytes = "1.5"
-chrono = "0.4.34"
+bytes = "1.6"
+chrono = "0.4.38"
 ctor = "0.2.8"
+datafusion = "44"
 derive_builder = "0.20"
 either = "1"
 env_logger = "0.11.0"
-fnv = "1"
+fnv = "1.0.7"
 futures = "0.3"
 iceberg = { version = "0.4.0", path = "./crates/iceberg" }
 iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" }
 iceberg-catalog-hms = { version = "0.4.0", path = "./crates/catalog/hms" }
 iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" }
 iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" }
 itertools = "0.13"
-log = "0.4"
+log = "0.4.22"
 mockito = "1"
 murmur3 = "0.5.2"
 num-bigint = "0.4.6"
-once_cell = "1"
-opendal = "0.51.0"
+once_cell = "1.19"
+opendal = "0.51.1"
 ordered-float = "4"
-parquet = "53.1"
-paste = "1"
+parquet = "53.4.0"
+paste = "1.0.15"
 pilota = "0.11.2"
 pretty_assertions = "1.4"
 port_scanner = "0.1.5"
-rand = "0.8"
+rand = "0.8.5"
 regex = "1.10.5"
-reqwest = { version = "0.12", default-features = false, features = ["json"] }
+reqwest = { version = "0.12.2", default-features = false, features = ["json"] }
 rust_decimal = "1.31"
-serde = { version = "1", features = ["rc"] }
-serde_bytes = "0.11.8"
-serde_derive = "1"
-serde_json = "1"
+serde = { version = "1.0.204", features = ["rc"] }
+serde_bytes = "0.11.15"
+serde_derive = "1.0.204"
+serde_json = "1.0.138"
 serde_repr = "0.1.16"
 serde_with = "3.4"
-tempfile = "3.8"
-tokio = { version = "1", default-features = false }
+tempfile = "3.16"
+tokio = { version = "1.36", default-features = false }
 typed-builder = "0.20"
-url = "2"
+url = "2.2.2"
 urlencoding = "2"
-uuid = { version = "1.6.1", features = ["v7"] }
+uuid = { version = "1.12.1", features = ["v7"] }
 volo-thrift = "0.10"
 hive_metastore = "0.1"
 tera = "1"

bindings/python/pyproject.toml

@@ -23,6 +23,7 @@ build-backend = "maturin"
 name = "pyiceberg_core"
 version = "0.4.0"
 readme = "project-description.md"
+requires-python = "~=3.9"
 classifiers = [
   "Development Status :: 4 - Beta",
   "Intended Audience :: Developers",

crates/catalog/glue/src/catalog.rs

@@ -20,7 +20,9 @@ use std::fmt::Debug;
 
 use async_trait::async_trait;
 use aws_sdk_glue::types::TableInput;
-use iceberg::io::FileIO;
+use iceberg::io::{
+    FileIO, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN,
+};
 use iceberg::spec::{TableMetadata, TableMetadataBuilder};
 use iceberg::table::Table;
 use iceberg::{
@@ -34,7 +36,9 @@ use crate::utils::{
     convert_to_database, convert_to_glue_table, convert_to_namespace, create_metadata_location,
     create_sdk_config, get_default_table_location, get_metadata_location, validate_namespace,
 };
-use crate::with_catalog_id;
+use crate::{
+    with_catalog_id, AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN,
+};
 
 #[derive(Debug, TypedBuilder)]
 /// Glue Catalog configuration
@@ -69,11 +73,40 @@ impl GlueCatalog {
     /// Create a new glue catalog
     pub async fn new(config: GlueCatalogConfig) -> Result<Self> {
         let sdk_config = create_sdk_config(&config.props, config.uri.as_ref()).await;
+        let mut file_io_props = config.props.clone();
+        if !file_io_props.contains_key(S3_ACCESS_KEY_ID) {
+            if let Some(access_key_id) = file_io_props.get(AWS_ACCESS_KEY_ID) {
+                file_io_props.insert(S3_ACCESS_KEY_ID.to_string(), access_key_id.to_string());
+            }
+        }
+        if !file_io_props.contains_key(S3_SECRET_ACCESS_KEY) {
+            if let Some(secret_access_key) = file_io_props.get(AWS_SECRET_ACCESS_KEY) {
+                file_io_props.insert(
+                    S3_SECRET_ACCESS_KEY.to_string(),
+                    secret_access_key.to_string(),
+                );
+            }
+        }
+        if !file_io_props.contains_key(S3_REGION) {
+            if let Some(region) = file_io_props.get(AWS_REGION_NAME) {
+                file_io_props.insert(S3_REGION.to_string(), region.to_string());
+            }
+        }
+        if !file_io_props.contains_key(S3_SESSION_TOKEN) {
+            if let Some(session_token) = file_io_props.get(AWS_SESSION_TOKEN) {
+                file_io_props.insert(S3_SESSION_TOKEN.to_string(), session_token.to_string());
+            }
+        }
+        if !file_io_props.contains_key(S3_ENDPOINT) {
+            if let Some(aws_endpoint) = config.uri.as_ref() {
+                file_io_props.insert(S3_ENDPOINT.to_string(), aws_endpoint.to_string());
+            }
+        }
 
         let client = aws_sdk_glue::Client::new(&sdk_config);
 
         let file_io = FileIO::from_path(&config.warehouse)?
-            .with_props(&config.props)
+            .with_props(file_io_props)
             .build()?;
 
         Ok(GlueCatalog {

crates/catalog/glue/tests/glue_catalog_test.rs

@@ -44,7 +44,7 @@ fn before_all() {
         normalize_test_name(module_path!()),
         format!("{}/testdata/glue_catalog", env!("CARGO_MANIFEST_DIR")),
     );
-    docker_compose.run();
+    docker_compose.up();
     guard.replace(docker_compose);
 }
 

crates/catalog/hms/tests/hms_catalog_test.rs

@@ -43,7 +43,7 @@ fn before_all() {
         normalize_test_name(module_path!()),
         format!("{}/testdata/hms_catalog", env!("CARGO_MANIFEST_DIR")),
     );
-    docker_compose.run();
+    docker_compose.up();
     guard.replace(docker_compose);
 }
 

crates/catalog/rest/Cargo.toml

@@ -35,7 +35,7 @@ chrono = { workspace = true }
 http = "1.1.0"
 iceberg = { workspace = true }
 itertools = { workspace = true }
-log = "0.4.20"
+log = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }

crates/catalog/rest/tests/rest_catalog_test.rs

@@ -41,7 +41,7 @@ fn before_all() {
         normalize_test_name(module_path!()),
         format!("{}/testdata/rest_catalog", env!("CARGO_MANIFEST_DIR")),
     );
-    docker_compose.run();
+    docker_compose.up();
     guard.replace(docker_compose);
 }
 

crates/catalog/s3tables/Cargo.toml

@@ -20,7 +20,7 @@ name = "iceberg-catalog-s3tables"
 version = { workspace = true }
 edition = { workspace = true }
 homepage = { workspace = true }
-rust-version = { workspace = true }
+rust-version = "1.81.0"
 
 categories = ["database"]
 description = "Apache Iceberg Rust S3Tables Catalog"
@@ -32,9 +32,10 @@ keywords = ["iceberg", "sql", "catalog"]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 aws-config = { workspace = true }
-aws-sdk-s3tables = "1.2.0"
+aws-sdk-s3tables = "1.8.0"
 iceberg = { workspace = true }
 serde_json = { workspace = true }
+typed-builder = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 
 [dev-dependencies]

crates/catalog/s3tables/src/catalog.rs

@@ -31,11 +31,12 @@ use iceberg::{
     Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, TableCreation,
     TableIdent,
 };
+use typed_builder::TypedBuilder;
 
 use crate::utils::{create_metadata_location, create_sdk_config};
 
 /// S3Tables catalog configuration.
-#[derive(Debug)]
+#[derive(Debug, TypedBuilder)]
 pub struct S3TablesCatalogConfig {
     /// Unlike other buckets, S3Tables bucket is not a physical bucket, but a virtual bucket
     /// that is managed by s3tables. We can't directly access the bucket with path like
@@ -48,8 +49,10 @@ pub struct S3TablesCatalogConfig {
     /// - `aws_access_key_id`: The AWS access key ID to use.
     /// - `aws_secret_access_key`: The AWS secret access key to use.
     /// - `aws_session_token`: The AWS session token to use.
+    #[builder(default)]
     properties: HashMap<String, String>,
     /// Endpoint URL for the catalog.
+    #[builder(default, setter(strip_option(fallback = endpoint_url_opt)))]
     endpoint_url: Option<String>,
 }
 
@@ -515,12 +518,9 @@ mod tests {
             None => return Ok(None),
         };
 
-        let properties = HashMap::new();
-        let config = S3TablesCatalogConfig {
-            table_bucket_arn,
-            properties,
-            endpoint_url: None,
-        };
+        let config = S3TablesCatalogConfig::builder()
+            .table_bucket_arn(table_bucket_arn)
+            .build();
 
         Ok(Some(S3TablesCatalog::new(config).await?))
     }

crates/catalog/s3tables/src/lib.rs

@@ -21,3 +21,5 @@
 
 mod catalog;
 mod utils;
+
+pub use catalog::*;

crates/catalog/sql/Cargo.toml

@@ -40,7 +40,7 @@ uuid = { workspace = true, features = ["v4"] }
 iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
 itertools = { workspace = true }
 regex = "1.10.5"
-sqlx = { version = "0.8.0", features = [
+sqlx = { version = "0.8.1", features = [
   "tls-rustls",
   "runtime-tokio",
   "any",

crates/examples/Cargo.toml

@@ -27,7 +27,7 @@ rust-version = { workspace = true }
 [dependencies]
 iceberg = { workspace = true }
 iceberg-catalog-rest = { workspace = true }
-tokio = { version = "1", features = ["full"] }
+tokio = { workspace = true, features = ["full"] }
 
 [[example]]
 name = "rest-catalog-namespace"