Merge branch 'main' into reduce_expr_size

Author: zhuqi-lucas

.github/actions/setup-macos-aarch64-builder/action.yaml

@@ -45,5 +45,7 @@ runs:
         rustup component add rustfmt
     - name: Setup rust cache
       uses: Swatinem/rust-cache@v2
+      with:
+           save-if: ${{ github.ref_name == 'main' }}
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime        

.github/workflows/rust.yml

@@ -51,6 +51,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs
+          save-if: ${{ github.ref_name == 'main' }}
       - name: Prepare cargo build
         run: |
           # Adding `--locked` here to assert that the `Cargo.lock` file is up to
@@ -99,6 +104,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: false # set in linux-test
+          shared-key: "amd-ci"
       - name: Check datafusion-substrait (default features)
         run: cargo check --profile ci --all-targets -p datafusion-substrait
         #
@@ -162,6 +172,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: false # set in linux-test
+          shared-key: "amd-ci"
       - name: Check datafusion (default features)
         run: cargo check --profile ci --all-targets -p datafusion
       #
@@ -247,15 +262,22 @@ jobs:
     name: cargo test (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
       - name: Setup Rust toolchain
-        run: rustup toolchain install stable
-      - name: Install Protobuf Compiler
-        run: sudo apt-get install -y protobuf-compiler
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+         save-if: ${{ github.ref_name == 'main' }}
+         shared-key: "amd-ci"
       - name: Run tests (excluding doctests and datafusion-cli)
         env:
           RUST_BACKTRACE: 1
@@ -279,6 +301,10 @@ jobs:
     name: cargo test datafusion-cli (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
+    # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged
+    # and cache should be added
+    # container:
+    #  image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
@@ -329,6 +355,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.ref_name == 'main' }}
+          shared-key: "amd-ci-linux-test-example"
       - name: Run examples
         run: |
           # test datafusion-sql examples
@@ -655,6 +686,11 @@ jobs:
           rust-version: stable
       - name: Install Clippy
         run: rustup component add clippy
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.ref_name == 'main' }}
+          shared-key: "amd-ci-clippy"
       - name: Run clippy
         run: ci/scripts/rust_clippy.sh
 

Cargo.lock

@@ -114,8 +114,6 @@ imdb:                   Join Order Benchmark (JOB) using the IMDB dataset conver
 
 # Micro-Benchmarks (specific operators and features)
 cancellation:           How long cancelling a query takes
-parquet:                Benchmark of parquet reader's filtering speed
-sort:                   Benchmark of sorting speed
 
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 Supported Configuration (Environment Variables)
@@ -303,8 +301,6 @@ main() {
                     run_tpch "10" "csv"
                     run_tpch_mem "10"
                     run_cancellation
-                    run_parquet
-                    run_sort
                     run_clickbench_1
                     run_clickbench_partitioned
                     run_clickbench_extended
@@ -338,12 +334,6 @@ main() {
                 cancellation)
                     run_cancellation
                     ;;
-                parquet)
-                    run_parquet
-                    ;;
-                sort)
-                    run_sort
-                    ;;
                 clickbench_1)
                     run_clickbench_1
                     ;;
@@ -528,22 +518,6 @@ run_cancellation() {
     debug_run $CARGO_COMMAND --bin dfbench -- cancellation --iterations 5 --path "${DATA_DIR}/cancellation" -o "${RESULTS_FILE}"
 }
 
-# Runs the parquet filter benchmark
-run_parquet() {
-    RESULTS_FILE="${RESULTS_DIR}/parquet.json"
-    echo "RESULTS_FILE: ${RESULTS_FILE}"
-    echo "Running parquet filter benchmark..."
-    debug_run $CARGO_COMMAND --bin parquet -- filter --path "${DATA_DIR}" --scale-factor 1.0 --iterations 5 -o "${RESULTS_FILE}"
-}
-
-# Runs the sort benchmark
-run_sort() {
-    RESULTS_FILE="${RESULTS_DIR}/sort.json"
-    echo "RESULTS_FILE: ${RESULTS_FILE}"
-    echo "Running sort benchmark..."
-    debug_run $CARGO_COMMAND --bin parquet -- sort --path "${DATA_DIR}" --scale-factor 1.0 --iterations 5 -o "${RESULTS_FILE}"
-}
-
 
 # Downloads the single file hits.parquet ClickBench datasets from
 # https://github.com/ClickHouse/ClickBench/tree/main#data-loading

benchmarks/bench.sh

@@ -33,9 +33,7 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-use datafusion_benchmarks::{
-    cancellation, clickbench, h2o, imdb, parquet_filter, sort, sort_tpch, tpch,
-};
+use datafusion_benchmarks::{cancellation, clickbench, h2o, imdb, sort_tpch, tpch};
 
 #[derive(Debug, StructOpt)]
 #[structopt(about = "benchmark command")]
@@ -44,8 +42,6 @@ enum Options {
     Clickbench(clickbench::RunOpt),
     H2o(h2o::RunOpt),
     Imdb(imdb::RunOpt),
-    ParquetFilter(parquet_filter::RunOpt),
-    Sort(sort::RunOpt),
     SortTpch(sort_tpch::RunOpt),
     Tpch(tpch::RunOpt),
     TpchConvert(tpch::ConvertOpt),
@@ -61,8 +57,6 @@ pub async fn main() -> Result<()> {
         Options::Clickbench(opt) => opt.run().await,
         Options::H2o(opt) => opt.run().await,
         Options::Imdb(opt) => Box::pin(opt.run()).await,
-        Options::ParquetFilter(opt) => opt.run().await,
-        Options::Sort(opt) => opt.run().await,
         Options::SortTpch(opt) => opt.run().await,
         Options::Tpch(opt) => Box::pin(opt.run()).await,
         Options::TpchConvert(opt) => opt.run().await,

benchmarks/src/bin/dfbench.rs

@@ -20,8 +20,6 @@ pub mod cancellation;
 pub mod clickbench;
 pub mod h2o;
 pub mod imdb;
-pub mod parquet_filter;
-pub mod sort;
 pub mod sort_tpch;
 pub mod tpch;
 pub mod util;