Merge branch 'main' into fd-extend-converter

.cargo/audit.toml

@@ -21,4 +21,8 @@ ignore = [
   # Marvin Attack: potential key recovery through timing sidechannels
   # Issues: https://github.com/apache/iceberg-rust/issues/221
   "RUSTSEC-2023-0071",
+  # `derivative` is unmaintained; consider using an alternative
+  #
+  # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293
+  "RUSTSEC-2024-0388",
 ]

.github/actions/setup-builder/action.yml

@@ -21,18 +21,24 @@ name: Prepare Rust Builder
 description: 'Prepare Rust Build Environment'
 inputs:
   rust-version:
-    description: 'version of rust to install (e.g. stable)'
-    required: true
-    default: 'stable'
+    description: 'version of rust to install and use'
 runs:
   using: "composite"
   steps:
-    - name: Setup Rust toolchain
+    - name: Setup specified Rust toolchain
       shell: bash
+      if: ${{ inputs.rust-version != '' }}
       run: |
         echo "Installing ${{ inputs.rust-version }}"
         rustup toolchain install ${{ inputs.rust-version }}
-        rustup default ${{ inputs.rust-version }}
+        rustup override set ${{ inputs.rust-version }}
+        rustup component add rustfmt clippy
+    - name: Setup Rust toolchain according to rust-toolchain.toml
+      shell: bash
+      if: ${{ inputs.rust-version == '' }}
+      run: |
+        echo "Installing toolchain according to rust-toolchain.toml"
+        rustup show
         rustup component add rustfmt clippy
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766

.github/workflows/audit.yml

@@ -32,12 +32,14 @@ on:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
 
+  schedule:
+    - cron: '0 0 * * *'
+
 jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Install cargo-audit
-        run: cargo install cargo-audit
-      - name: Run audit check
-        run: cargo audit
+      - uses: rustsec/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -43,14 +43,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
       - name: Check License Header
         uses: apache/skywalking-eyes/[email protected]
 
-      - name: Install cargo-sort
-        run: make install-cargo-sort
-
       - name: Install taplo-cli
-        run: make install-taplo-cli
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
+      - name: Check toml format
+        run: make check-toml
 
       - name: Cargo format
         run: make check-fmt
@@ -61,11 +65,19 @@ jobs:
       - name: Cargo clippy
         run: make check-clippy
 
+      - name: Install cargo-sort
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
       - name: Cargo sort
-        run: make cargo-sort
+        run: cargo sort -c -w
 
+      - name: Install cargo-machete
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-machete
       - name: Cargo Machete
-        run: make cargo-machete
+        run: cargo machete
 
   build:
     runs-on: ${{ matrix.os }}
@@ -80,9 +92,10 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
 
+      - name: Cache Rust artifacts
+        uses: Swatinem/rust-cache@v2
+
       - name: Build
         run: make build
 
@@ -96,6 +109,13 @@ jobs:
           - windows-latest
     steps:
       - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
+      - name: Cache Rust artifacts
+        uses: Swatinem/rust-cache@v2
+
       - name: Build
         run: cargo build -p iceberg --no-default-features
 
@@ -106,8 +126,9 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
+
+      - name: Cache Rust artifacts
+        uses: Swatinem/rust-cache@v2
 
       - name: Test
         run: cargo test --no-fail-fast --all-targets --all-features --workspace
@@ -117,3 +138,24 @@ jobs:
 
       - name: Doc Test
         run: cargo test --no-fail-fast --doc --all-features --workspace
+
+  msrv:
+    name: Verify MSRV
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Nightly Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Generate minimal versions lockfile
+        run: |
+          cargo generate-lockfile -Z direct-minimal-versions -Z minimal-versions
+          # Some dependencies don't correctly specify a minimal version for their dependencies and will fail to build.
+          # So we update these transitive dependencies here.
+          cargo update tap faststr metainfo linkedbytes
+      - name: Setup MSRV Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
+      - name: Check MSRV
+        run: |
+          cargo +${{ env.rust_msrv }} check --locked --workspace --exclude iceberg-datafusion --exclude iceberg-catalog-s3tables

.github/workflows/ci_typos.yml

@@ -42,4 +42,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.28.3
+        uses: crate-ci/typos@v1.29.4

.gitignore

@@ -16,7 +16,6 @@
 # under the License.
 
 target
-Cargo.lock
 .idea
 .vscode
 **/.DS_Store

.licenserc.yaml

@@ -31,4 +31,5 @@ header:
     - '**/DEPENDENCIES.*.tsv'
     # Release distributions
     - 'dist/*'
+    - 'Cargo.lock'
   comment: on-failure

CHANGELOG.md

@@ -160,6 +160,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 * Prep 0.4.0 release by @sungwy in https://github.com/apache/iceberg-rust/pull/809
 * feat: Add RemovePartitionSpecs table update by @c-thiel in https://github.com/apache/iceberg-rust/pull/804
 * feat: Store file io props to allow re-build it by @Xuanwo in https://github.com/apache/iceberg-rust/pull/802
+* chore: chmod +x on `verify.py` script by @sungwy in https://github.com/apache/iceberg-rust/pull/817
+* fix: Reading a table with positional deletes should fail by @Fokko in https://github.com/apache/iceberg-rust/pull/826
 
 ## [v0.3.0] - 2024-08-14
 

CONTRIBUTING.md

@@ -116,7 +116,7 @@ cargo 1.69.0 (6e9a83356 2023-04-12)
 
 Currently, iceberg-rust uses Docker to set up environment for integration tests. Native Docker has some limitations, please check (https://github.com/apache/iceberg-rust/pull/748). Please use Orbstack or Podman.
 
-For MacOS users, you can install [OrbStack](https://orbstack.dev/) as a docker alternative.
+For MacOS users, you can install [OrbStack as a docker alternative](docs/contributing/orbstack.md).
 
 For podman, refer to [Using Podman instead of Docker](docs/contributing/podman.md)