Merge remote-tracking branch 'upstream/main' into feat/metrics

Author: gotocoding-DB

CHANGELOG.md

@@ -1,5 +1,68 @@
 # Changelog
 
+## [v0.18.0](https://github.com/delta-io/delta-kernel-rs/tree/v0.18.0/) (2025-11-19)
+
+[Full Changelog](https://github.com/delta-io/delta-kernel-rs/compare/v0.17.1...v0.18.0)
+
+### 🏗️ Breaking changes
+1. New Engine StorageHandler head API ([#1465])
+   - Engine API implementers must add the `head` API to StorageHandler which fetches metadata about a file in storage
+2. Add remove_files API ([#1353])
+   - The schema for scan rows (from `Scan::scan_metadata`) has been updated to include two
+     new fields: `fileConstantValues.tags` and `fileConstantValues.defaultRowCommitVersion`.
+
+### 🚀 Features / new APIs
+
+1. Add parser for iceberg compat properties ([#1466])
+2. Pass ColumnMappingMode to physical_name ([#1403])
+3. Allow visiting entire domain metadata ([#1384])
+4. Add Table Feature Info ([#1462])
+5. *(FFI)* Snapshot log tail FFI ([#1379])
+6. Add generic is_feature_supported and is_feature_enabled methods to TableConfiguration ([#1405])
+7. Un-deprecate ArrayData.array_elements() ([#1493])
+8. Allow writes to CDF tables for add-only, remove-only, and non-data-change transactions ([#1490])
+9. *(catalog-managed)* UCCommitter ([#1418])
+
+### 🐛 Bug Fixes
+
+1. Eliminate endless busy looping in read_json_files on failed read ([#1489])
+2. Handle array/map types in ffi schema example and test ([#1497])
+
+### 📚 Documentation
+
+1. Fix docs for rustc 1.92+ ([#1470])
+
+### 🚜 Refactor
+
+1. Harmonize checkpoint and log compaction iterators ([#1436])
+2. Avoid overly complex itertools methods in log listing code ([#1434])
+3. Simplify creation of default engine in tests ([#1437])
+
+### 🧪 Testing
+
+1. Add tests for StructField.physical_name ([#1469])
+
+[#1466]: https://github.com/delta-io/delta-kernel-rs/pull/1466
+[#1403]: https://github.com/delta-io/delta-kernel-rs/pull/1403
+[#1465]: https://github.com/delta-io/delta-kernel-rs/pull/1465
+[#1436]: https://github.com/delta-io/delta-kernel-rs/pull/1436
+[#1470]: https://github.com/delta-io/delta-kernel-rs/pull/1470
+[#1384]: https://github.com/delta-io/delta-kernel-rs/pull/1384
+[#1462]: https://github.com/delta-io/delta-kernel-rs/pull/1462
+[#1474]: https://github.com/delta-io/delta-kernel-rs/pull/1474
+[#1379]: https://github.com/delta-io/delta-kernel-rs/pull/1379
+[#1434]: https://github.com/delta-io/delta-kernel-rs/pull/1434
+[#1437]: https://github.com/delta-io/delta-kernel-rs/pull/1437
+[#1353]: https://github.com/delta-io/delta-kernel-rs/pull/1353
+[#1489]: https://github.com/delta-io/delta-kernel-rs/pull/1489
+[#1405]: https://github.com/delta-io/delta-kernel-rs/pull/1405
+[#1469]: https://github.com/delta-io/delta-kernel-rs/pull/1469
+[#1493]: https://github.com/delta-io/delta-kernel-rs/pull/1493
+[#1497]: https://github.com/delta-io/delta-kernel-rs/pull/1497
+[#1490]: https://github.com/delta-io/delta-kernel-rs/pull/1490
+[#1418]: https://github.com/delta-io/delta-kernel-rs/pull/1418
+
+
 ## [v0.17.1](https://github.com/delta-io/delta-kernel-rs/tree/v0.17.1/) (2025-11-13)
 
 [Full Changelog](https://github.com/delta-io/delta-kernel-rs/compare/v0.17.0...v0.17.1)

Cargo.toml

@@ -27,4 +27,4 @@ license = "Apache-2.0"
 repository = "https://github.com/delta-io/delta-kernel-rs"
 readme = "README.md"
 rust-version = "1.85"
-version = "0.17.1"
+version = "0.18.0"

README.md

@@ -52,10 +52,10 @@ consumer's own `Engine` trait, the kernel has a feature flag to enable a default
 ```toml
 # fewer dependencies, requires consumer to implement Engine trait.
 # allows consumers to implement their own in-memory format
-delta_kernel = "0.17.1"
+delta_kernel = "0.18.0"
 
 # or turn on the default engine, based on latest arrow
-delta_kernel = { version = "0.17.1", features = ["default-engine", "arrow"] }
+delta_kernel = { version = "0.18.0", features = ["default-engine", "arrow"] }
 ```
 
 ### Feature flags

acceptance/src/data.rs

@@ -4,11 +4,12 @@ use delta_kernel::arrow::array::{Array, RecordBatch};
 use delta_kernel::arrow::compute::{concat_batches, lexsort_to_indices, take, SortColumn};
 use delta_kernel::arrow::datatypes::{DataType, Schema};
 
+use delta_kernel::engine::arrow_data::EngineDataArrowExt as _;
 use delta_kernel::parquet::arrow::async_reader::{
     ParquetObjectReader, ParquetRecordBatchStreamBuilder,
 };
 use delta_kernel::snapshot::Snapshot;
-use delta_kernel::{engine::arrow_data::ArrowEngineData, DeltaResult, Engine, Error};
+use delta_kernel::{DeltaResult, Engine, Error};
 use futures::{stream::TryStreamExt, StreamExt};
 use itertools::Itertools;
 use object_store::{local::LocalFileSystem, ObjectStore};
@@ -119,11 +120,7 @@ pub async fn assert_scan_metadata(
     let batches: Vec<RecordBatch> = scan
         .execute(engine)?
         .map(|data| -> DeltaResult<_> {
-            let record_batch: RecordBatch = data?
-                .into_any()
-                .downcast::<ArrowEngineData>()
-                .unwrap()
-                .into();
+            let record_batch = data?.try_into_record_batch()?;
             if schema.is_none() {
                 schema = Some(record_batch.schema());
             }

ffi/Cargo.toml

@@ -25,7 +25,7 @@ url = "2"
 delta_kernel = { path = "../kernel", default-features = false, features = [
   "internal-api",
 ] }
-delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.17.1" }
+delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.18.0" }
 
 [build-dependencies]
 cbindgen = "0.29.2"

ffi/src/engine_data.rs

@@ -1,5 +1,4 @@
 //! EngineData related ffi code
-
 #[cfg(feature = "default-engine-base")]
 use delta_kernel::arrow;
 #[cfg(feature = "default-engine-base")]
@@ -8,7 +7,7 @@ use delta_kernel::arrow::array::{
     ArrayData, RecordBatch, StructArray,
 };
 #[cfg(feature = "default-engine-base")]
-use delta_kernel::engine::arrow_data::ArrowEngineData;
+use delta_kernel::engine::arrow_data::{ArrowEngineData, EngineDataArrowExt as _};
 #[cfg(feature = "default-engine-base")]
 use delta_kernel::DeltaResult;
 use delta_kernel::EngineData;
@@ -96,11 +95,7 @@ pub unsafe extern "C" fn get_raw_arrow_data(
 // TODO: This method leaks the returned pointer memory. How will the engine free it?
 #[cfg(feature = "default-engine-base")]
 fn get_raw_arrow_data_impl(data: Box<dyn EngineData>) -> DeltaResult<*mut ArrowFFIData> {
-    let record_batch: delta_kernel::arrow::array::RecordBatch = data
-        .into_any()
-        .downcast::<ArrowEngineData>()
-        .map_err(|_| delta_kernel::Error::EngineDataType("ArrowEngineData".to_string()))?
-        .into();
+    let record_batch = data.try_into_record_batch()?;
     let sa: StructArray = record_batch.into();
     let array_data: ArrayData = sa.into();
     // these call `clone`. is there a way to not copy anything and what exactly are they cloning?

ffi/src/table_changes.rs

@@ -3,9 +3,9 @@
 use std::sync::Arc;
 use std::sync::Mutex;
 
-use delta_kernel::arrow::array::{Array, ArrayData, RecordBatch, StructArray};
+use delta_kernel::arrow::array::{Array, ArrayData, StructArray};
 use delta_kernel::arrow::ffi::to_ffi;
-use delta_kernel::engine::arrow_data::ArrowEngineData;
+use delta_kernel::engine::arrow_data::EngineDataArrowExt;
 use delta_kernel::table_changes::scan::TableChangesScan;
 use delta_kernel::table_changes::TableChanges;
 use delta_kernel::EngineData;
@@ -319,11 +319,7 @@ fn scan_table_changes_next_impl(data: &ScanTableChangesIterator) -> DeltaResult<
         return Ok(ArrowFFIData::empty());
     };
 
-    let record_batch: RecordBatch = data
-        .into_any()
-        .downcast::<ArrowEngineData>()
-        .map_err(|_| delta_kernel::Error::EngineDataType("ArrowEngineData".to_string()))?
-        .into();
+    let record_batch = data.try_into_record_batch()?;
 
     let batch_struct_array: StructArray = record_batch.into();
     let array_data: ArrayData = batch_struct_array.into_data();
@@ -346,6 +342,7 @@ mod tests {
     use delta_kernel::arrow::record_batch::RecordBatch;
     use delta_kernel::arrow::util::pretty::pretty_format_batches;
     use delta_kernel::engine::arrow_conversion::TryIntoArrow as _;
+    use delta_kernel::engine::arrow_data::ArrowEngineData;
     use delta_kernel::engine::default::DefaultEngine;
     use delta_kernel::schema::{DataType, StructField, StructType};
     use delta_kernel::Engine;
@@ -355,7 +352,7 @@ mod tests {
     use std::sync::Arc;
     use test_utils::{
         actions_to_string_with_metadata, add_commit, generate_batch, record_batch_to_bytes,
-        to_arrow, IntoArray as _, TestAction,
+        IntoArray as _, TestAction,
     };
 
     const PARQUET_FILE1: &str =
@@ -480,7 +477,7 @@ mod tests {
     ) -> DeltaResult<Vec<RecordBatch>> {
         let scan_results = scan.execute(engine)?;
         scan_results
-            .map(|data| -> DeltaResult<_> { to_arrow(data?) })
+            .map(EngineDataArrowExt::try_into_record_batch)
             .try_collect()
     }
 
@@ -699,7 +696,7 @@ mod tests {
             }
             let engine_data =
                 ok_or_panic(unsafe { get_engine_data(data.array, &data.schema, allocate_err) });
-            let record_batch = unsafe { to_arrow(engine_data.into_inner()) }?;
+            let record_batch = unsafe { engine_data.into_inner().try_into_record_batch() }?;
 
             println!("Batch ({i}) num rows {:?}", record_batch.num_rows());
             batches.push(record_batch);

kernel/Cargo.toml

@@ -39,7 +39,7 @@ pre-release-hook = [
 ]
 
 [dependencies]
-delta_kernel_derive = { path = "../derive-macros", version = "0.17.1" }
+delta_kernel_derive = { path = "../derive-macros", version = "0.18.0" }
 bytes = "1.10"
 chrono = "0.4.41"
 crc = "3.2.2"

kernel/examples/read-table-changes/src/main.rs

@@ -4,7 +4,7 @@ use clap::Parser;
 use common::{LocationArgs, ParseWithExamples};
 use delta_kernel::arrow::array::RecordBatch;
 use delta_kernel::arrow::util::pretty::print_batches;
-use delta_kernel::engine::arrow_data::ArrowEngineData;
+use delta_kernel::engine::arrow_data::EngineDataArrowExt;
 use delta_kernel::table_changes::TableChanges;
 use delta_kernel::DeltaResult;
 use itertools::Itertools;
@@ -38,14 +38,7 @@ fn main() -> DeltaResult<()> {
     let table_changes_scan = table_changes.into_scan_builder().build()?;
     let batches: Vec<RecordBatch> = table_changes_scan
         .execute(Arc::new(engine))?
-        .map(|data| -> DeltaResult<_> {
-            let record_batch: RecordBatch = data?
-                .into_any()
-                .downcast::<ArrowEngineData>()
-                .map_err(|_| delta_kernel::Error::EngineDataType("ArrowEngineData".to_string()))?
-                .into();
-            Ok(record_batch)
-        })
+        .map(EngineDataArrowExt::try_into_record_batch)
         .try_collect()?;
     print_batches(&batches)?;
     Ok(())

kernel/examples/read-table-multi-threaded/src/main.rs

@@ -9,10 +9,10 @@ use arrow::record_batch::RecordBatch;
 use arrow::util::pretty::print_batches;
 use common::{LocationArgs, ParseWithExamples, ScanArgs};
 use delta_kernel::actions::deletion_vector::split_vector;
-use delta_kernel::engine::arrow_data::ArrowEngineData;
+use delta_kernel::engine::arrow_data::EngineDataArrowExt as _;
 use delta_kernel::scan::state::{transform_to_logical, DvInfo, Stats};
 use delta_kernel::schema::SchemaRef;
-use delta_kernel::{DeltaResult, Engine, EngineData, ExpressionRef, FileMeta, Snapshot};
+use delta_kernel::{DeltaResult, Engine, ExpressionRef, FileMeta, Snapshot};
 
 use clap::Parser;
 use url::Url;
@@ -59,15 +59,6 @@ struct ScanFile {
     dv_info: DvInfo,
 }
 
-// we know we're using arrow under the hood, so cast an EngineData into something we can work with
-fn to_arrow(data: Box<dyn EngineData>) -> DeltaResult<RecordBatch> {
-    Ok(data
-        .into_any()
-        .downcast::<ArrowEngineData>()
-        .map_err(|_| delta_kernel::Error::EngineDataType("ArrowEngineData".to_string()))?
-        .into())
-}
-
 // This is the callback that will be called for each valid scan row
 fn send_scan_file(
     scan_tx: &mut spmc::Sender<ScanFile>,
@@ -231,7 +222,7 @@ fn do_work(
             )
             .unwrap();
 
-            let record_batch = to_arrow(logical).unwrap();
+            let record_batch = logical.try_into_record_batch().unwrap();
 
             // need to split the dv_mask. what's left in dv_mask covers this result, and rest
             // will cover the following results