chore: Update to Rust Polars 0.48 (#141)

stijnherfst · web-flow · commit 3157958b4b29 · 2025-05-21T10:07:33.000+02:00
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -31,7 +31,7 @@ jobs:
       - name: Set up Rust
         run: rustup show
 
-      - uses: mozilla-actions/sccache-action@v0.0.3
+      - uses: mozilla-actions/sccache-action@v0.0.8
 
       - run: cargo test
         working-directory: pyo3-polars
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,24 +9,24 @@ members = [
 ]
 
 [workspace.dependencies]
-polars = { version = "0.46.0", default-features = false }
-polars-core = { version = "0.46.0", default-features = false }
-polars-arrow = { version = "0.46.0", default-features = false }
-polars-ffi = { version = "0.46.0", default-features = false }
-polars-plan = { version = "0.46.0", default-features = false }
-polars-lazy = { version = "0.46.0", default-features = false }
-polars-python = {version = "0.46.0", default-features = false }
-polars-utils = {version = "0.46.0", default-features = false}
+polars = { version = "0.48.0", default-features = false }
+polars-core = { version = "0.48.0", default-features = false }
+polars-arrow = { version = "0.48.0", default-features = false }
+polars-ffi = { version = "0.48.0", default-features = false }
+polars-plan = { version = "0.48.0", default-features = false }
+polars-lazy = { version = "0.48.0", default-features = false }
+polars-python = { version = "0.48.0", default-features = false }
+polars-utils = { version = "0.48.0", default-features = false }
 
 [workspace.dependencies.arrow]
 package = "polars-arrow"
-version = "0.46.0"
+version = "0.48.0"
 path = "../polars/crates/polars-arrow"
 default-features = false
 
-#[patch.crates-io]
-#polars = { git = "https://github.com/pola-rs/polars.git" }
-#polars-core = { git = "https://github.com/pola-rs/polars.git" }
-#polars-ffi = { git = "https://github.com/pola-rs/polars.git" }
-#polars-plan = { git = "https://github.com/pola-rs/polars.git" }
-#polars-lazy = { git = "https://github.com/pola-rs/polars.git" }
+# [patch.crates-io]
+# polars = { git = "https://github.com/pola-rs/polars.git" }
+# polars-core = { git = "https://github.com/pola-rs/polars.git" }
+# polars-ffi = { git = "https://github.com/pola-rs/polars.git" }
+# polars-plan = { git = "https://github.com/pola-rs/polars.git" }
+# polars-lazy = { git = "https://github.com/pola-rs/polars.git" }
diff --git a/example/derive_expression/expression_lib/Cargo.toml b/example/derive_expression/expression_lib/Cargo.toml
@@ -9,10 +9,10 @@ name = "expression_lib"
 crate-type = ["cdylib"]
 
 [dependencies]
-polars = { workspace = true, features = ["fmt", "dtype-date", "timezones"], default-features = false }
+num = "*"
+polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "object"], default-features = false }
 polars-arrow = { workspace = true, default-features = false }
-pyo3 = { version = "0.23", features = ["abi3-py38"] }
+pyo3 = { version = "0.24.2", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] }
 rayon = "1.7.0"
 serde = { version = "1", features = ["derive"] }
-num = "*"
diff --git a/example/derive_expression/expression_lib/pyproject.toml b/example/derive_expression/expression_lib/pyproject.toml
@@ -4,6 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "expression_lib"
+version = "0.0.1"
 requires-python = ">=3.8"
 classifiers = [
   "Programming Language :: Rust",
diff --git a/example/derive_expression/expression_lib/src/distances.rs b/example/derive_expression/expression_lib/src/distances.rs
@@ -25,6 +25,7 @@ fn jacc_helper<T: NativeType + Hash + Eq>(a: &PrimitiveArray<T>, b: &PrimitiveAr
     s3_len as f64 / (s1.len() + s2.len() - s3_len) as f64
 }
 
+#[allow(unexpected_cfgs)]
 pub(super) fn naive_jaccard_sim(a: &ListChunked, b: &ListChunked) -> PolarsResult<Float64Chunked> {
     polars_ensure!(
         a.inner_dtype() == b.inner_dtype(),
diff --git a/example/derive_expression/expression_lib/src/expressions.rs b/example/derive_expression/expression_lib/src/expressions.rs
@@ -193,7 +193,10 @@ struct TimeZone {
 
 fn convert_timezone(input_fields: &[Field], kwargs: TimeZone) -> PolarsResult<Field> {
     FieldsMapper::new(input_fields).try_map_dtype(|dtype| match dtype {
-        DataType::Datetime(tu, _) => Ok(DataType::Datetime(*tu, Some(kwargs.tz.into()))),
+        DataType::Datetime(tu, _) => Ok(DataType::Datetime(
+            *tu,
+            datatypes::TimeZone::opt_try_new(Some(kwargs.tz))?,
+        )),
         _ => polars_bail!(ComputeError: "expected datetime"),
     })
 }
@@ -206,6 +209,11 @@ fn change_time_zone(input: &[Series], kwargs: TimeZone) -> PolarsResult<Series>
     let ca = input.datetime()?;
 
     let mut out = ca.clone();
-    out.set_time_zone(kwargs.tz.into())?;
+
+    let Some(timezone) = datatypes::TimeZone::opt_try_new(Some(kwargs.tz))? else {
+        polars_bail!(ComputeError: "expected timezone")
+    };
+
+    out.set_time_zone(timezone)?;
     Ok(out.into_series())
 }
diff --git a/example/derive_expression/requirements.txt b/example/derive_expression/requirements.txt
@@ -1,2 +1,2 @@
 maturin
-polars
+polars>=1.30.0b1
diff --git a/example/extend_polars_python_dispatch/extend_polars/Cargo.toml b/example/extend_polars_python_dispatch/extend_polars/Cargo.toml
@@ -9,9 +9,9 @@ name = "extend_polars"
 crate-type = ["cdylib"]
 
 [dependencies]
-polars = { workspace = true, features = ["fmt"] }
+polars = { workspace = true, features = ["fmt", "object"] }
 polars-core = { workspace = true }
 polars-lazy = { workspace = true }
-pyo3 = { version = "0.23", features = ["extension-module"] }
+pyo3 = { version = "0.24.2", features = ["extension-module"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] }
 rayon = "1.10"
diff --git a/example/extend_polars_python_dispatch/requirements.txt b/example/extend_polars_python_dispatch/requirements.txt
@@ -1,2 +1,2 @@
 maturin
-polars[pyarrow]
+polars[pyarrow]>=1.30.0b1
diff --git a/example/io_plugin/io_plugin/Cargo.toml b/example/io_plugin/io_plugin/Cargo.toml
@@ -9,8 +9,8 @@ name = "io_plugin"
 crate-type = ["cdylib"]
 
 [dependencies]
-polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy"], default-features = false }
+polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy", "object", "strings"], default-features = false }
 polars-arrow = { workspace = true, default-features = false }
-pyo3 = { version = "0.23", features = ["abi3-py38"] }
+pyo3 = { version = "0.24.2", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive", "lazy"] }
 rand = { version = "0.8.5", features = [] }
diff --git a/example/io_plugin/io_plugin/io_plugin/__init__.py b/example/io_plugin/io_plugin/io_plugin/__init__.py
@@ -42,4 +42,4 @@ def source_generator(
 
     # create src again to compute the schema
     src = RandomSource(samplers, 0, 0)
-    return register_io_source(callable=source_generator, schema=src.schema())
+    return register_io_source(io_source=source_generator, schema=src.schema())
diff --git a/example/io_plugin/io_plugin/pyproject.toml b/example/io_plugin/io_plugin/pyproject.toml
@@ -4,6 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "io_plugin"
+version = "0.0.1"
 requires-python = ">=3.10"
 classifiers = [
   "Programming Language :: Rust",
diff --git a/example/io_plugin/io_plugin/src/samplers.rs b/example/io_plugin/io_plugin/src/samplers.rs
@@ -52,7 +52,7 @@ where
     }
 
     fn dtype(&self) -> DataType {
-        T::PolarsType::get_dtype()
+        T::PolarsType::get_static_dtype()
     }
 
     fn next_n(&mut self, n: usize) -> Series {
diff --git a/example/io_plugin/requirements.txt b/example/io_plugin/requirements.txt
@@ -1,2 +1,2 @@
 maturin
-polars
+polars>=1.30.0b1
diff --git a/pyo3-polars-derive/Cargo.toml b/pyo3-polars-derive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars-derive"
-version = "0.14.0"
+version = "0.15.0"
 edition = "2021"
 license = "MIT"
 readme = "README.md"
@@ -17,13 +17,14 @@ name = "tests"
 path = "tests/run.rs"
 
 [dependencies]
+polars-arrow = { workspace = true }
 polars-core = { workspace = true }
 polars-ffi = { workspace = true }
 polars-plan = { workspace = true }
-polars-arrow = { workspace = true }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = { version = "2", features = ["full", "extra-traits"] }
 
 [dev-dependencies]
+pyo3-polars = { version = "0.21.0", path = "../pyo3-polars", features = ["derive"] }
 trybuild = { version = "1", features = ["diff"] }
diff --git a/pyo3-polars-derive/tests/01.rs b/pyo3-polars-derive/tests/01.rs
@@ -1,17 +1,19 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::{Field, Series};
+use polars_core::prelude::*;
 use polars_plan::dsl::FieldsMapper;
 use pyo3_polars_derive::polars_expr;
 
 fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult<Field> {
     FieldsMapper::new(input_fields).map_to_supertype()
 }
 
-#[polars_expr(type_func=horizontal_product_output)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+#[polars_expr(output_type_func=horizontal_product_output)]
+fn horizontal_product(series: &[Series], kwargs: Option<&str>) -> PolarsResult<Series> {
+    let _ = kwargs;
+
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?
     }
     Ok(acc)
 }
diff --git a/pyo3-polars-derive/tests/02.rs b/pyo3-polars-derive/tests/02.rs
@@ -1,12 +1,14 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::Series;
+use polars_core::prelude::*;
 use pyo3_polars_derive::polars_expr;
 
 #[polars_expr(output_type=Int32)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+fn horizontal_product(series: &[Series], kwargs: Option<&str>) -> PolarsResult<Series> {
+    let _ = kwargs;
+
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?
     }
     Ok(acc)
 }
diff --git a/pyo3-polars/Cargo.toml b/pyo3-polars/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars"
-version = "0.20.0"
+version = "0.21.0"
 edition = "2021"
 license = "MIT"
 readme = "../README.md"
@@ -12,22 +12,22 @@ description = "Expression plugins and PyO3 types for polars"
 [dependencies]
 libc = "0.2" # pyo3 depends on libc already, so this does not introduce an extra dependence.
 once_cell = "1"
-polars = { workspace = true, default-features = true}
+polars = { workspace = true, default-features = true }
 polars-arrow = { workspace = true, default-features = false }
 polars-core = { workspace = true, default-features = false }
 polars-ffi = { workspace = true, optional = true }
 polars-lazy = { workspace = true, optional = true }
 polars-plan = { workspace = true, optional = true }
-polars-utils = {workspace = true, features = ["serde"], optional = true }
-pyo3 = "0.23"
-pyo3-polars-derive = { version = "0.14.0", path = "../pyo3-polars-derive", optional = true }
+polars-utils = { workspace = true, features = ["serde"], optional = true }
+pyo3 = "0.24.2"
+pyo3-polars-derive = { version = "0.15.0", path = "../pyo3-polars-derive", optional = true }
 serde = { version = "1", optional = true }
 serde-pickle = { version = "1", optional = true }
 thiserror = "1"
 
 [features]
 # Polars python is needed because all variants need to be acttivated of the DSL.
-lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "polars-utils", "polars-lazy/python"]
+lazy = ["polars/serde-lazy", "polars/object", "polars-plan", "polars-lazy/serde", "polars-utils", "polars-lazy/python"]
 derive = ["pyo3-polars-derive", "polars-plan", "polars-ffi", "serde-pickle", "serde"]
 dtype-full = ["polars/dtype-full", "dtype-decimal", "dtype-array", "dtype-struct", "dtype-categorical"]
 object = ["polars/object"]
diff --git a/pyo3-polars/src/error.rs b/pyo3-polars/src/error.rs
@@ -2,7 +2,9 @@ use std::fmt::{Debug, Formatter};
 
 use polars::prelude::PolarsError;
 use pyo3::create_exception;
-use pyo3::exceptions::{PyException, PyIOError, PyIndexError, PyRuntimeError, PyValueError};
+use pyo3::exceptions::{
+    PyAssertionError, PyException, PyIOError, PyIndexError, PyRuntimeError, PyValueError,
+};
 use pyo3::prelude::*;
 use thiserror::Error;
 
@@ -16,8 +18,9 @@ pub enum PyPolarsErr {
 
 impl std::convert::From<PyPolarsErr> for PyErr {
     fn from(err: PyPolarsErr) -> PyErr {
-        fn convert(err: &PolarsError) -> PyErr {
+        fn convert(err: PolarsError) -> PyErr {
             match err {
+                PolarsError::AssertionError(error) => PyAssertionError::new_err(error.to_string()),
                 PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
                 PolarsError::NoData(err) => NoDataError::new_err(err.to_string()),
                 PolarsError::ShapeMismatch(err) => ShapeError::new_err(err.to_string()),
@@ -38,14 +41,16 @@ impl std::convert::From<PyPolarsErr> for PyErr {
                 }
                 PolarsError::SQLInterface(err) => SQLInterface::new_err(err.to_string()),
                 PolarsError::SQLSyntax(err) => SQLSyntax::new_err(err.to_string()),
-                PolarsError::Context { error, .. } => convert(error),
+                PolarsError::Context { error, .. } => convert(*error),
+                #[cfg(feature = "lazy")]
+                PolarsError::Python { error } => error.0,
             }
         }
 
         use PyPolarsErr::*;
-        match &err {
+        match err {
             Polars(err) => convert(err),
-            _ => PyRuntimeError::new_err(format!("{:?}", &err)),
+            _ => PyRuntimeError::new_err(format!("{err:?}")),
         }
     }
 }
@@ -60,6 +65,7 @@ impl Debug for PyPolarsErr {
     }
 }
 
+create_exception!(exceptions, AssertionError, PyException);
 create_exception!(exceptions, ColumnNotFound, PyException);
 create_exception!(exceptions, SchemaFieldNotFound, PyException);
 create_exception!(exceptions, StructFieldNotFound, PyException);
diff --git a/pyo3-polars/src/types.rs b/pyo3-polars/src/types.rs
diff --git a/rust-toolchain.toml b/rust-toolchain.toml

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ fn jacc_helper<T: NativeType + Hash + Eq>(a: &PrimitiveArray<T>, b: &PrimitiveAr`
`25`	`25`	`s3_len as f64 / (s1.len() + s2.len() - s3_len) as f64`
`26`	`26`	`}`
`27`	`27`
	`28`	`+#[allow(unexpected_cfgs)]`
`28`	`29`	`pub(super) fn naive_jaccard_sim(a: &ListChunked, b: &ListChunked) -> PolarsResult<Float64Chunked> {`
`29`	`30`	`polars_ensure!(`
`30`	`31`	`a.inner_dtype() == b.inner_dtype(),`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`maturin`
`2`		`-polars`
	`2`	`+polars>=1.30.0b1`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`maturin`
`2`		`-polars[pyarrow]`
	`2`	`+polars[pyarrow]>=1.30.0b1`