[WIP] Upgrade to V1.2.1 (#462)

Maxxen · luong-hq · yutannihilation · web-flow · commit 64300764501f · 2025-03-12T10:44:31.000+01:00
* fix: pin duckdb and libduckdb-sys versions Signed-off-by: luong-hq <luong@hyperquery.ai> * Bump arrow to the version compatible with chrono 0.4.40 * Update only arrow * Fix minor typo * Do not pass stdlib * Fix #380, bug in code that reads Enums In turning a ValueRef to a Value, the code was indexing with the row index directly into the dictionary's string array, so iterating any query's results was effectively just listing the Enum variants in order until row index > number of variants and it panicked. Fix by getting the row value (key) and using that to index into the dictionary's string array. * Fix incorrect Enum test The test tests for correct value extraction, based on built-in examples in DuckDB. The old expected value was flatly incorrect, as can be seen in duckdb directly with `SELECT medium_enum FROM test_all_types();`. * Fix writing Arrow Decimal128 with zero scale * Fix CI install of rustfmt and clippy Extra space breaks hecrj/setup-rust-action. Arguments should be comma-separated. * update duckdb to v1.2.1 * replace rust toolchain action * fix warnings * add back sanitizer * move logicaltypeid * add correct component for asan * add back extensions to asan build * bundle ICU as part of extensions-full --------- Signed-off-by: luong-hq <luong@hyperquery.ai> Co-authored-by: luong-hq <luong@hyperquery.ai> Co-authored-by: Hiroaki Yutani <yutani.ini@gmail.com> Co-authored-by: Shritesh Bhattarai <shr@ite.sh> Co-authored-by: Nickolas Fotopoulos <nickolas.fotopoulos@gmail.com> Co-authored-by: sgrebnov <sergei.grebnov@gmail.com>
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -11,12 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      # This has a matcher for test panics, so we use it even though elsewhere
-      # we use actions-rs/toolchain.
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable${{ matrix.host }}
-          targets: ${{ matrix.target }}
+          toolchain: stable
+          target: ${{ matrix.target }}
           components: 'rustfmt, clippy'
 
       # cargo publish
diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml
@@ -26,20 +26,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
-      # This has a matcher for test panics, so we use it even though elsewhere
-      # we use actions-rs/toolchain.
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable${{ matrix.host }}
-          targets: ${{ matrix.target }}
+          toolchain: stable
+          target: ${{ matrix.target }}
           components: 'rustfmt, clippy'
 
       # download libduckdb
       - uses: robinraju/release-downloader@v1.4
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v1.2.0"
+          tag: "v1.2.1"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 
@@ -112,9 +110,41 @@ jobs:
           key: index-${{ runner.os }}-${{ github.run_number }}
           restore-keys: |
             index-${{ runner.os }}-
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable
-          targets: x86_64-pc-windows-msvc
+          toolchain: stable
+          target: x86_64-pc-windows-msvc
 
-      - run: cargo install cargo-examples
+      - run: cargo install cargo-examples
+
+  Sanitizer:
+    name: Address Sanitizer
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # Need nightly rust.
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+          components: 'rust-src'
+
+      - name: Tests with asan
+        env:
+          RUSTFLAGS: -Zsanitizer=address -C debuginfo=0
+          RUSTDOCFLAGS: -Zsanitizer=address
+          ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=1"
+          # Work around https://github.com/rust-lang/rust/issues/59125 by
+          # disabling backtraces. In an ideal world we'd probably suppress the
+          # leak sanitization, but we don't care about backtraces here, so long
+          # as the other tests have them.
+          RUST_BACKTRACE: "0"
+        run: |
+          cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu --package duckdb
+      - name: publish crates --dry-run
+        uses: katyo/publish-crates@v2
+        with:
+          path: './'
+          args: --allow-dirty --all-features
+          dry-run: true
+          ignore-unpublished-changes: true
diff --git a/Cargo.toml b/Cargo.toml
@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.2.0"
+version = "1.2.1"
 authors = ["wangfenjin <wangfenj@gmail.com>"]
 edition = "2021"
 repository = "https://github.com/duckdb/duckdb-rs"
@@ -19,9 +19,9 @@ license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "1.2.0", path = "crates/duckdb" }
-libduckdb-sys = { version = "1.2.0", path = "crates/libduckdb-sys" }
-duckdb-loadable-macros = { version = "0.1.4", path = "crates/duckdb-loadable-macros" }
+duckdb = { version = "=1.2.1", path = "crates/duckdb" }
+libduckdb-sys = { version = "=1.2.1", path = "crates/libduckdb-sys" }
+duckdb-loadable-macros = { version = "=0.1.5", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
 bindgen = { version = "0.71.1", default-features = false }
 byteorder = "1.3"
@@ -63,4 +63,4 @@ unicase = "2.6.0"
 url = "2.1"
 uuid = "1.0"
 vcpkg = "0.2"
-arrow = { version = "54", default-features = false }
+arrow = { version = "54.2.1", default-features = false }
diff --git a/crates/duckdb-loadable-macros/Cargo.toml b/crates/duckdb-loadable-macros/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "duckdb-loadable-macros"
-version = "0.1.4"
+version = "0.1.5"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
diff --git a/crates/duckdb/Cargo.toml b/crates/duckdb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "1.2.0"
+version = "1.2.1"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -22,6 +22,7 @@ default = []
 bundled = ["libduckdb-sys/bundled"]
 json = ["libduckdb-sys/json", "bundled"]
 parquet = ["libduckdb-sys/parquet", "bundled"]
+icu = ["libduckdb-sys/icu", "bundled"]
 vscalar = []
 vscalar-arrow = []
 vtab = []
@@ -30,7 +31,7 @@ vtab-excel = ["vtab", "calamine"]
 vtab-arrow = ["vtab", "num"]
 appender-arrow = ["vtab-arrow"]
 vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
-extensions-full = ["json", "parquet", "vtab-full"]
+extensions-full = ["json", "parquet", "icu", "vtab-full"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
 polars = ["dep:polars", "dep:polars-arrow"]
diff --git a/crates/duckdb/src/lib.rs b/crates/duckdb/src/lib.rs
@@ -126,7 +126,7 @@ pub mod types;
 #[cfg(feature = "vtab")]
 pub mod vtab;
 
-/// The duckdb table function interface
+/// The duckdb scalar function interface
 #[cfg(feature = "vscalar")]
 pub mod vscalar;
 
diff --git a/crates/duckdb/src/test_all_types.rs b/crates/duckdb/src/test_all_types.rs
@@ -651,7 +651,7 @@ fn test_single(idx: &mut i32, column: String, value: ValueRef) {
         },
         "medium_enum" => match idx {
             0 => assert_eq!(value.to_owned(), Value::Enum("enum_0".to_string())),
-            1 => assert_eq!(value.to_owned(), Value::Enum("enum_1".to_string())),
+            1 => assert_eq!(value.to_owned(), Value::Enum("enum_299".to_string())),
             _ => assert_eq!(value, ValueRef::Null),
         },
         "large_enum" => match idx {
diff --git a/crates/duckdb/src/types/string.rs b/crates/duckdb/src/types/string.rs
@@ -6,6 +6,7 @@ pub struct DuckString<'a> {
 }
 
 impl<'a> DuckString<'a> {
+    #[allow(dead_code)]
     pub(crate) fn new(ptr: &'a mut duckdb_string_t) -> Self {
         DuckString { ptr }
     }
diff --git a/crates/duckdb/src/types/value_ref.rs b/crates/duckdb/src/types/value_ref.rs
@@ -6,8 +6,8 @@ use rust_decimal::prelude::*;
 
 use arrow::{
     array::{
-        Array, ArrayRef, DictionaryArray, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray,
-        UnionArray,
+        Array, ArrayRef, DictionaryArray, FixedSizeListArray, LargeListArray, ListArray, MapArray, StringArray,
+        StructArray, UnionArray,
     },
     datatypes::{UInt16Type, UInt32Type, UInt8Type},
 };
@@ -239,22 +239,21 @@ impl From<ValueRef<'_>> for Value {
                 }
             },
             ValueRef::Enum(items, idx) => {
-                let value = Row::value_ref_internal(
-                    idx,
-                    0,
-                    match items {
-                        EnumType::UInt8(res) => res.values(),
-                        EnumType::UInt16(res) => res.values(),
-                        EnumType::UInt32(res) => res.values(),
-                    },
-                )
-                .to_owned();
-
-                if let Value::Text(s) = value {
-                    Value::Enum(s)
-                } else {
-                    panic!("Enum value is not a string")
+                let dict_values = match items {
+                    EnumType::UInt8(res) => res.values(),
+                    EnumType::UInt16(res) => res.values(),
+                    EnumType::UInt32(res) => res.values(),
+                }
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .expect("Enum value is not a string");
+                let dict_key = match items {
+                    EnumType::UInt8(res) => res.key(idx),
+                    EnumType::UInt16(res) => res.key(idx),
+                    EnumType::UInt32(res) => res.key(idx),
                 }
+                .unwrap();
+                Value::Enum(dict_values.value(dict_key).to_string())
             }
             ValueRef::Struct(items, idx) => {
                 let value: Vec<(String, Value)> = items
diff --git a/crates/duckdb/src/vscalar/mod.rs b/crates/duckdb/src/vscalar/mod.rs
@@ -14,7 +14,7 @@ use crate::{
 };
 mod function;
 
-/// The duckdb Arrow table function interface
+/// The duckdb Arrow scalar function interface
 #[cfg(feature = "vscalar-arrow")]
 pub mod arrow;
 
@@ -32,7 +32,7 @@ pub trait VScalar: Sized {
     ///
     /// This function is unsafe because it:
     ///
-    /// - Dereferences multiple raw pointers (`func``).
+    /// - Dereferences multiple raw pointers (`func`).
     ///
     unsafe fn invoke(
         state: &Self::State,
diff --git a/crates/duckdb/src/vtab/arrow.rs b/crates/duckdb/src/vtab/arrow.rs
@@ -1,11 +1,12 @@
-use super::{BindInfo, DataChunkHandle, InitInfo, LogicalTypeHandle, LogicalTypeId, TableFunctionInfo, VTab};
+use super::{BindInfo, DataChunkHandle, InitInfo, LogicalTypeHandle, TableFunctionInfo, VTab};
 use std::sync::Arc;
 use std::sync::{atomic::AtomicBool, Mutex};
 
 use crate::{
-    core::{ArrayVector, FlatVector, Inserter, ListVector, StructVector, Vector},
+    core::{ArrayVector, FlatVector, Inserter, ListVector, LogicalTypeId, StructVector, Vector},
     types::DuckString,
 };
+
 use arrow::array::as_map_array;
 use arrow::{
     array::{
@@ -201,8 +202,12 @@ pub fn to_duckdb_logical_type(data_type: &DataType) -> Result<LogicalTypeHandle,
             &to_duckdb_logical_type(child.data_type())?,
             *array_size as u64,
         )),
-        DataType::Decimal128(width, scale) if *scale > 0 => {
-            // DuckDB does not support negative decimal scales
+        DataType::Decimal128(width, scale) => {
+            if *scale < 0 {
+                return Err(
+                    format!("Unsupported data type: {data_type}, negative decimal scale is not supported").into(),
+                );
+            }
             Ok(LogicalTypeHandle::decimal(*width, (*scale).try_into().unwrap()))
         }
         DataType::Map(field, _) => arrow_map_to_duckdb_logical_type(field),
@@ -1610,14 +1615,21 @@ mod test {
 
     #[test]
     fn test_decimal128_roundtrip() -> Result<(), Box<dyn Error>> {
+        // With default width and scale
         let array: PrimitiveArray<arrow::datatypes::Decimal128Type> =
             Decimal128Array::from(vec![i128::from(1), i128::from(2), i128::from(3)]);
         check_rust_primitive_array_roundtrip(array.clone(), array)?;
 
-        // With width and scale
+        // With custom width and scale
         let array: PrimitiveArray<arrow::datatypes::Decimal128Type> =
             Decimal128Array::from(vec![i128::from(12345)]).with_data_type(DataType::Decimal128(5, 2));
         check_rust_primitive_array_roundtrip(array.clone(), array)?;
+
+        // With width and zero scale
+        let array: PrimitiveArray<arrow::datatypes::Decimal128Type> =
+            Decimal128Array::from(vec![i128::from(12345)]).with_data_type(DataType::Decimal128(5, 0));
+        check_rust_primitive_array_roundtrip(array.clone(), array)?;
+
         Ok(())
     }
 
diff --git a/crates/duckdb/src/vtab/excel.rs b/crates/duckdb/src/vtab/excel.rs
@@ -1,7 +1,7 @@
 use std::sync::atomic::{self, AtomicUsize};
 
-use super::{BindInfo, DataChunkHandle, InitInfo, LogicalTypeHandle, LogicalTypeId, TableFunctionInfo, VTab};
-use crate::core::Inserter;
+use super::{BindInfo, DataChunkHandle, InitInfo, LogicalTypeHandle, TableFunctionInfo, VTab};
+use crate::core::{Inserter, LogicalTypeId};
 use calamine::{open_workbook_auto, DataType, Range, Reader};
 
 #[repr(C)]
diff --git a/crates/duckdb/src/vtab/mod.rs b/crates/duckdb/src/vtab/mod.rs
@@ -23,7 +23,7 @@ mod excel;
 pub use function::{BindInfo, InitInfo, TableFunction, TableFunctionInfo};
 pub use value::Value;
 
-use crate::core::{DataChunkHandle, LogicalTypeHandle, LogicalTypeId};
+use crate::core::{DataChunkHandle, LogicalTypeHandle};
 use ffi::{duckdb_bind_info, duckdb_data_chunk, duckdb_function_info, duckdb_init_info};
 
 /// Given a raw pointer to a box, free the box and the data contained within it.
diff --git a/crates/libduckdb-sys/Cargo.toml b/crates/libduckdb-sys/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "libduckdb-sys"
-version = "1.2.0"
+version = "1.2.1"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
@@ -21,7 +21,8 @@ bundled = ["cc"]
 buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"]
 json = ["bundled"]
 parquet = ["bundled"]
-extensions-full = ["json", "parquet"]
+icu = ["bundled"]
+extensions-full = ["json", "parquet", "icu"]
 winduckdb = []
 # Warning: experimental feature
 loadable-extension = ["prettyplease", "quote", "syn"]
diff --git a/crates/libduckdb-sys/build.rs b/crates/libduckdb-sys/build.rs
@@ -133,6 +133,9 @@ mod build_bundled {
         #[cfg(feature = "json")]
         add_extension(&mut cfg, &manifest, "json", &mut cpp_files, &mut include_dirs);
 
+        #[cfg(feature = "icu")]
+        add_extension(&mut cfg, &manifest, "icu", &mut cpp_files, &mut include_dirs);
+
         // duckdb/tools/pythonpkg/setup.py
         cfg.define("DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT", "1");
         cfg.define("DUCKDB_EXTENSION_AUTOLOAD_DEFAULT", "1");
@@ -152,8 +155,6 @@ mod build_bundled {
 
         cfg.cpp(true)
             .flag_if_supported("-std=c++11")
-            .flag_if_supported("-stdlib=libc++")
-            .flag_if_supported("-stdlib=libstdc++")
             .flag_if_supported("/bigobj")
             .warnings(false)
             .flag_if_supported("-w");
diff --git a/crates/libduckdb-sys/duckdb-sources b/crates/libduckdb-sys/duckdb-sources
@@ -1 +1 @@
-Subproject commit 5f5512b827df6397afd31daedb4bbdee76520019
+Subproject commit 38cc952efb5440ae01a6d56ad41cae257315d652
diff --git a/crates/libduckdb-sys/duckdb.tar.gz b/crates/libduckdb-sys/duckdb.tar.gz
diff --git a/crates/libduckdb-sys/src/bindgen_bundled_version.rs b/crates/libduckdb-sys/src/bindgen_bundled_version.rs
diff --git a/crates/libduckdb-sys/src/bindgen_bundled_version_loadable.rs b/crates/libduckdb-sys/src/bindgen_bundled_version_loadable.rs
diff --git a/crates/libduckdb-sys/update_sources.py b/crates/libduckdb-sys/update_sources.py
diff --git a/crates/libduckdb-sys/upgrade.sh b/crates/libduckdb-sys/upgrade.sh

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ pub struct DuckString<'a> {`
`6`	`6`	`}`
`7`	`7`
`8`	`8`	`impl<'a> DuckString<'a> {`
	`9`	`+ #[allow(dead_code)]`
`9`	`10`	`pub(crate) fn new(ptr: &'a mut duckdb_string_t) -> Self {`
`10`	`11`	`DuckString { ptr }`
`11`	`12`	`}`