merge upstream main

.github/actions/setup-builder/action.yml

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is heavily inspired by
+#  [datafusion](https://github.com/apache/datafusion/blob/main/.github/actions/setup-builder/action.yaml).
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        echo "Installing ${{ inputs.rust-version }}"
+        rustup toolchain install ${{ inputs.rust-version }}
+        rustup default ${{ inputs.rust-version }}
+        rustup component add rustfmt clippy
+    - name: Fixup git permissions
+      # https://github.com/actions/checkout/issues/766
+      shell: bash
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.github/workflows/ci.yml

@@ -29,15 +29,29 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+env:
+  rust_msrv: "1.77.1"
+
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: Check License Header
         uses: apache/skywalking-eyes/[email protected]
 
+      - name: Install cargo-sort
+        run: make install-cargo-sort
+
+      - name: Install taplo-cli
+        run: make install-taplo-cli
+
       - name: Cargo format
         run: make check-fmt
 
@@ -50,8 +64,29 @@ jobs:
       - name: Cargo sort
         run: make cargo-sort
 
+      - name: Cargo Machete
+        run: make cargo-machete
 
   build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
+
+      - name: Build
+        run: make build
+
+  build_with_no_default_features:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -62,15 +97,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Build
-        run: cargo build
+        run: cargo build -p iceberg --no-default-features
 
   unit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
+
       - name: Test
         run: cargo test --no-fail-fast --all-targets --all-features --workspace
-
+
+      - name: Async-std Test
+        run: cargo test --no-fail-fast --all-targets --no-default-features  --features "async-std" --features "storage-fs" --workspace
+
       - name: Doc Test
         run: cargo test --no-fail-fast --doc --all-features --workspace

.github/workflows/ci_typos.yml

@@ -41,7 +41,5 @@ jobs:
       FORCE_COLOR: 1
     steps:
       - uses: actions/checkout@v4
-      - run: curl -LsSf https://github.com/crate-ci/typos/releases/download/v1.14.8/typos-v1.14.8-x86_64-unknown-linux-musl.tar.gz | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
-
-      - name: do typos check with typos-cli
-        run: typos
+      - name: Check typos
+        uses: crate-ci/[email protected]

.github/workflows/publish.yml

@@ -21,13 +21,11 @@ on:
   push:
     tags:
       - '*'
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/publish.yml"
   workflow_dispatch:
 
+env:
+  rust_msrv: "1.77.1"
+
 jobs:
   publish:
     runs-on: ubuntu-latest
@@ -42,9 +40,11 @@ jobs:
           - "crates/catalog/rest"
     steps:
       - uses: actions/checkout@v4
-      - name: Dryrun ${{ matrix.package }}
-        working-directory: ${{ matrix.package }}
-        run: cargo publish --all-features --dry-run
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
 
       - name: Publish ${{ matrix.package }}
         working-directory: ${{ matrix.package }}

.github/workflows/website.yml

@@ -47,6 +47,11 @@ jobs:
       - name: Copy asf file
         run: cp .asf.yaml ./website/book/.asf.yaml
 
+      - name: Build API docs
+        run: |
+          cargo doc --no-deps --workspace --all-features
+          cp -r target/doc ./website/book/api
+
       - name: Deploy to gh-pages
         uses: peaceiris/[email protected]
         if: github.event_name == 'push' && github.ref_name == 'main'

CONTRIBUTING.md

@@ -110,14 +110,16 @@ $ cargo version
 cargo 1.69.0 (6e9a83356 2023-04-12)
 ```
 
-#### Install docker
+#### Install Docker or Podman
 
-Currently, iceberg-rust uses docker to set up environment for integration tests.
+Currently, iceberg-rust uses Docker to set up environment for integration tests. Podman is also supported.
 
-You can learn how to install docker from [here](https://docs.docker.com/get-docker/).
+You can learn how to install Docker from [here](https://docs.docker.com/get-docker/).
 
 For macos users, you can install [OrbStack](https://orbstack.dev/) as a docker alternative.
 
+For podman users, refer to [Using Podman instead of Docker](docs/contributing/podman.md)
+
 ## Build
 
 * To compile the project: `make build`

Cargo.toml

@@ -38,19 +38,22 @@ rust-version = "1.77.1"
 anyhow = "1.0.72"
 apache-avro = "0.16"
 array-init = "2"
-arrow-arith = { version = "51" }
-arrow-array = { version = "51" }
-arrow-ord = { version = "51" }
-arrow-schema = { version = "51" }
-arrow-select = { version = "51" }
+arrow-arith = { version = "52" }
+arrow-array = { version = "52" }
+arrow-ord = { version = "52" }
+arrow-schema = { version = "52" }
+arrow-select = { version = "52" }
+arrow-string = { version = "52" }
 async-stream = "0.3.5"
 async-trait = "0.1"
+async-std = "1.12.0"
 aws-config = "1.1.8"
 aws-sdk-glue = "1.21.0"
 bimap = "0.6"
 bitvec = "1.0.1"
 bytes = "1.5"
 chrono = "0.4.34"
+ctor = "0.2.8"
 derive_builder = "0.20.0"
 either = "1"
 env_logger = "0.11.0"
@@ -60,17 +63,17 @@ iceberg = { version = "0.2.0", path = "./crates/iceberg" }
 iceberg-catalog-rest = { version = "0.2.0", path = "./crates/catalog/rest" }
 iceberg-catalog-hms = { version = "0.2.0", path = "./crates/catalog/hms" }
 itertools = "0.13"
-lazy_static = "1"
 log = "^0.4"
 mockito = "^1"
 murmur3 = "0.5.2"
 once_cell = "1"
 opendal = { git="https://github.com/twuebi/opendal.git", branch="tp/azdls-client-secret" }
 ordered-float = "4.0.0"
-parquet = "51"
-pilota = "0.11.0"
+parquet = "52"
+pilota = "0.11.2"
 pretty_assertions = "1.4.0"
 port_scanner = "0.1.5"
+regex = "1.10.5"
 reqwest = { version = "^0.12", default-features = false, features = ["json", "rustls-tls"] }
 rust_decimal = "1.31.0"
 serde = { version = "^1.0", features = ["rc"] }
@@ -81,11 +84,11 @@ serde_repr = "0.1.16"
 serde_with = "3.4.0"
 strum = "0.26.3"
 tempfile = "3.8"
-tokio = { version = "1", features = ["macros"] }
-typed-builder = "^0.18"
+tokio = { version = "1", default-features = false }
+typed-builder = "^0.19"
 url = "2"
 urlencoding = "2"
 uuid = { version = "1.6.1", features = ["v7"] }
-volo-thrift = "0.9.2"
-hive_metastore = "0.0.2"
+volo-thrift = "0.10"
+hive_metastore = "0.1.0"
 tera = "1"

Makefile

@@ -17,30 +17,37 @@
 
 .EXPORT_ALL_VARIABLES:
 
-RUST_LOG = debug
-
 build:
-	cargo build
+	cargo build --all-targets --all-features --workspace
 
 check-fmt:
-	cargo fmt --all -- --check
+	cargo  fmt --all -- --check
 
 check-clippy:
-	cargo clippy --all-targets --all-features --workspace -- -D warnings
+	cargo  clippy --all-targets --all-features --workspace -- -D warnings
+
+install-cargo-sort:
+	cargo install [email protected]
 
-cargo-sort:
-	cargo install cargo-sort
+cargo-sort: install-cargo-sort
 	cargo sort -c -w
 
-fix-toml:
-	cargo install taplo-cli --locked
+install-cargo-machete:
+	cargo install cargo-machete
+
+cargo-machete: install-cargo-machete
+	cargo machete
+
+install-taplo-cli:
+	cargo install [email protected]
+
+fix-toml: install-taplo-cli
 	taplo fmt
 
-check-toml:
-	cargo install taplo-cli --locked
+check-toml: install-taplo-cli
 	taplo check
 
-check: check-fmt check-clippy cargo-sort check-toml
+check: check-fmt check-clippy cargo-sort check-toml cargo-machete
 
 doc-test:
 	cargo test --no-fail-fast --doc --all-features --workspace