diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index c2e853b3..af0085d7 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -28,16 +28,17 @@ doc = false [dependencies] arrow = { workspace = true, features = ["pyarrow"] } +chrono = "0.4.38" datafusion = { workspace = true } datafusion-ffi = { workspace = true } futures = "0.3" paimon = { path = "../../crates/paimon", features = ["storage-all"] } paimon-datafusion = { path = "../../crates/integrations/datafusion", features = ["fulltext"] } -pyo3 = { version = "0.28", features = ["abi3-py310"] } +pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } serde_json = "1.0" tokio = { workspace = true } [dev-dependencies] # Enables `Python::attach` to auto-initialize an interpreter for Rust unit tests. # Only applied to the test build; the maturin/extension-module build does not pull dev-dependencies. -pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize"] } +pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize", "chrono"] } diff --git a/bindings/python/src/predicate.rs b/bindings/python/src/predicate.rs index cb2c8523..62d4b07e 100644 --- a/bindings/python/src/predicate.rs +++ b/bindings/python/src/predicate.rs @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder}; +use paimon::spec::{DataField, DataType, Datum, DecimalType, Predicate, PredicateBuilder}; use pyo3::exceptions::{PyNotImplementedError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDict, PyList, PyString}; +use pyo3::types::PyTzInfoAccess; +use pyo3::types::{PyBool, PyDateTime, PyDict, PyInt, PyList, PyString, PyTime, PyTzInfo}; /// Convert a single Python literal into a typed [`Datum`] driven by the target /// [`DataType`]. @@ -32,11 +33,24 @@ use pyo3::types::{PyBool, PyDict, PyList, PyString}; /// (which is an `int` subclass) and enforce the target range. /// - `Float`/`Double` accept Python `int` or `float` but reject `bool`. /// - `Char`/`VarChar` accept only a Python `str` (no implicit stringification). -/// - All other types (Date/Time/Timestamp/Decimal/Bytes/complex) are not -/// supported yet and raise `NotImplementedError`. +/// - `Date` accepts only a `datetime.date` that is not a `datetime.datetime` +/// (accepting the subclass would silently drop the time-of-day part). +/// - `Time` accepts only a naive `datetime.time` with whole-millisecond +/// microseconds (`Datum::Time` carries millis-of-day; sub-millisecond values +/// would be silently truncated). +/// - `Timestamp` accepts only a naive `datetime.datetime` (the type is +/// timezone-less); `LocalZonedTimestamp` accepts only a timezone-aware one +/// (converted to UTC). This mirrors the DataFusion pushdown path, which +/// refuses zoned literals for TIMESTAMP and naive ones for TIMESTAMP_LTZ. +/// - `Decimal` accepts a `decimal.Decimal` or an `int`, rescaled losslessly to +/// the column's scale; anything needing rounding, exceeding the column's +/// precision, non-finite, or a binary `float` is rejected. +/// - All other types (Bytes/complex) are not supported yet and raise +/// `NotImplementedError`. /// /// Errors: -/// - `ValueError` for type mismatches and out-of-range integers. +/// - `ValueError` for type mismatches, out-of-range integers, zoned/naive +/// timestamp mismatches, and lossy decimal/time conversions. /// - `NotImplementedError` for unsupported field types (message names the type). pub(crate) fn py_to_datum(value: &Bound<'_, PyAny>, data_type: &DataType) -> PyResult { match data_type { @@ -64,6 +78,11 @@ pub(crate) fn py_to_datum(value: &Bound<'_, PyAny>, data_type: &DataType) -> PyR .map_err(|_| PyValueError::new_err("expected a str literal for String field"))?; Ok(Datum::String(s.to_str()?.to_string())) } + DataType::Date(_) => date_datum(value), + DataType::Time(_) => time_datum(value), + DataType::Timestamp(_) => timestamp_datum(value), + DataType::LocalZonedTimestamp(_) => local_zoned_timestamp_datum(value), + DataType::Decimal(dec) => decimal_datum(value, dec), other => Err(PyNotImplementedError::new_err(format!( "literal conversion for type {other:?} is not supported yet" ))), @@ -103,6 +122,221 @@ fn float_val(value: &Bound<'_, PyAny>) -> PyResult { .map_err(|_| PyValueError::new_err("expected a numeric literal")) } +/// Convert a `datetime.date` into `Datum::Date` (epoch days). +/// +/// `datetime.datetime` is a `date` subclass but is rejected: accepting it would +/// silently drop the time-of-day part. +fn date_datum(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Err(PyValueError::new_err( + "expected a datetime.date literal for Date field, got datetime.datetime \ + (the time-of-day part would be dropped)", + )); + } + let date: chrono::NaiveDate = value + .extract() + .map_err(|_| PyValueError::new_err("expected a datetime.date literal for Date field"))?; + let epoch = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); + let days = date.signed_duration_since(epoch).num_days(); + let days = i32::try_from(days) + .map_err(|_| PyValueError::new_err(format!("date literal {date} out of range")))?; + Ok(Datum::Date(days)) +} + +/// Convert a naive `datetime.time` into `Datum::Time` (millis of day). +/// +/// Timezone-aware times are rejected (TIME has no timezone), as are +/// sub-millisecond microseconds (`Datum::Time` carries millis; truncating +/// would silently change comparison semantics). +fn time_datum(value: &Bound<'_, PyAny>) -> PyResult { + let time = value + .cast::() + .map_err(|_| PyValueError::new_err("expected a datetime.time literal for Time field"))?; + if time.get_tzinfo().is_some() { + return Err(PyValueError::new_err( + "expected a naive datetime.time literal for Time field (tzinfo must be None)", + )); + } + let time: chrono::NaiveTime = value + .extract() + .map_err(|_| PyValueError::new_err("expected a datetime.time literal for Time field"))?; + use chrono::Timelike; + if !time.nanosecond().is_multiple_of(1_000_000) { + return Err(PyValueError::new_err( + "time literal has sub-millisecond microseconds, which TIME comparisons \ + cannot represent without truncation", + )); + } + let millis = time.num_seconds_from_midnight() * 1_000 + time.nanosecond() / 1_000_000; + Ok(Datum::Time(millis as i32)) +} + +/// Split epoch microseconds into `(millis, nanos-of-milli)` with a Euclidean +/// split so pre-epoch instants keep a non-negative nano remainder, matching +/// `BinaryRow::get_timestamp_raw` and the DataFusion pushdown conversion. +fn micros_to_millis_nanos(micros: i64) -> (i64, i32) { + ( + micros.div_euclid(1_000), + (micros.rem_euclid(1_000) * 1_000) as i32, + ) +} + +/// Convert a naive `datetime.datetime` into `Datum::Timestamp`. +/// +/// TIMESTAMP (without time zone) is timezone-less, so timezone-aware datetimes +/// are rejected rather than silently reinterpreted — parity with the DataFusion +/// pushdown path, which refuses zoned literals for this type. +fn timestamp_datum(value: &Bound<'_, PyAny>) -> PyResult { + let dt = value.cast::().map_err(|_| { + PyValueError::new_err("expected a datetime.datetime literal for Timestamp field") + })?; + if dt.get_tzinfo().is_some() { + return Err(PyValueError::new_err( + "expected a naive datetime.datetime literal for Timestamp field (tzinfo must \ + be None); TIMESTAMP has no timezone — use a TIMESTAMP WITH LOCAL TIME ZONE \ + column for zoned instants", + )); + } + let naive: chrono::NaiveDateTime = value.extract().map_err(|_| { + PyValueError::new_err("expected a datetime.datetime literal for Timestamp field") + })?; + let (millis, nanos) = micros_to_millis_nanos(naive.and_utc().timestamp_micros()); + Ok(Datum::Timestamp { millis, nanos }) +} + +/// Convert a timezone-aware `datetime.datetime` into `Datum::LocalZonedTimestamp` +/// (the UTC instant). +/// +/// Aware follows Python's definition: `tzinfo` is set AND `utcoffset()` is not +/// `None`. A tzinfo whose `utcoffset()` returns `None` is naive — passing it to +/// `astimezone` would interpret the value as process-local time, normalizing +/// the same literal differently per machine. Naive datetimes are rejected +/// rather than assumed to be in any timezone — parity with the DataFusion +/// pushdown path, which refuses naive literals for TIMESTAMP WITH LOCAL TIME +/// ZONE. +fn local_zoned_timestamp_datum(value: &Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + let dt = value.cast::().map_err(|_| { + PyValueError::new_err("expected a datetime.datetime literal for LocalZonedTimestamp field") + })?; + // Python's aware/naive rule (datetime docs): aware iff tzinfo is not None + // and utcoffset() does not return None. + let aware = + dt.get_tzinfo().is_some() && !dt.call_method0(pyo3::intern!(py, "utcoffset"))?.is_none(); + if !aware { + return Err(PyValueError::new_err( + "expected a timezone-aware datetime.datetime literal for LocalZonedTimestamp \ + field (naive datetimes, including tzinfo with utcoffset() = None, are \ + ambiguous)", + )); + } + // Normalize through `astimezone(utc)` so any tzinfo implementation + // (zoneinfo, pytz, fixed offsets) is handled by Python itself. + let utc = PyTzInfo::utc(py)?; + let as_utc = dt.call_method1(pyo3::intern!(py, "astimezone"), (utc,))?; + let instant: chrono::DateTime = as_utc.extract()?; + let (millis, nanos) = micros_to_millis_nanos(instant.timestamp_micros()); + Ok(Datum::LocalZonedTimestamp { millis, nanos }) +} + +/// Convert a `decimal.Decimal` or `int` literal into `Datum::Decimal` at the +/// column's precision and scale. +/// +/// The conversion is exact: values that would need rounding at the column's +/// scale, or whose unscaled magnitude exceeds the column's precision, are +/// rejected. Binary `float`s are rejected outright (they are not exact decimal +/// values), as are non-finite decimals (NaN/Infinity). +fn decimal_datum(value: &Bound<'_, PyAny>, dec: &DecimalType) -> PyResult { + let precision = dec.precision(); + let scale = dec.scale(); + if value.is_instance_of::() { + return Err(PyValueError::new_err("bool is not a valid decimal literal")); + } + + let unscaled = if value.is_instance_of::() { + let v: i128 = value.extract().map_err(|_| { + PyValueError::new_err("int literal out of supported range for Decimal field") + })?; + v.checked_mul(10i128.pow(scale)).ok_or_else(|| { + PyValueError::new_err("int literal out of supported range for Decimal field") + })? + } else { + let decimal_cls = value.py().import("decimal")?.getattr("Decimal")?; + if !value.is_instance(&decimal_cls)? { + return Err(PyValueError::new_err( + "expected a decimal.Decimal or int literal for Decimal field (float is \ + not an exact decimal value)", + )); + } + decimal_to_unscaled(value, scale)? + }; + + // 10^precision fits i128 for the supported precision range (<= 38). + if unscaled.unsigned_abs() >= 10u128.pow(precision) { + return Err(PyValueError::new_err(format!( + "decimal literal does not fit DECIMAL({precision}, {scale})" + ))); + } + Ok(Datum::Decimal { + unscaled, + precision, + scale, + }) +} + +/// Rescale a finite `decimal.Decimal` to `scale` exactly via `as_tuple()` +/// (sign, digits, exponent), avoiding Decimal arithmetic whose context +/// precision could silently round. +fn decimal_to_unscaled(value: &Bound<'_, PyAny>, scale: u32) -> PyResult { + let tuple = value.call_method0("as_tuple")?; + let sign: u8 = tuple.getattr("sign")?.extract()?; + // For NaN/Infinity the exponent is a string ('n'/'N'/'F') and extraction fails. + let exponent: i64 = tuple.getattr("exponent")?.extract().map_err(|_| { + PyValueError::new_err("non-finite Decimal (NaN/Infinity) is not a valid literal") + })?; + let digits: Vec = tuple.getattr("digits")?.extract()?; + + let overflow = + || PyValueError::new_err("decimal literal out of supported range for Decimal field"); + let mut magnitude: i128 = 0; + for d in digits { + magnitude = magnitude + .checked_mul(10) + .and_then(|m| m.checked_add(d as i128)) + .ok_or_else(overflow)?; + } + + // value = ±magnitude * 10^exponent; unscaled = ±magnitude * 10^(exponent + scale). + let shift = exponent + scale as i64; + let magnitude = if shift >= 0 { + u32::try_from(shift) + .ok() + .filter(|s| *s <= 38) + .and_then(|s| magnitude.checked_mul(10i128.pow(s))) + .ok_or_else(overflow)? + } else if magnitude == 0 { + 0 + } else { + let down = u32::try_from(-shift).ok().filter(|s| *s <= 38).ok_or_else( + // More than 38 digits would have to be truncated; magnitude != 0 + // means that is always lossy. + || rounding_error(scale), + )?; + let divisor = 10i128.pow(down); + if magnitude % divisor != 0 { + return Err(rounding_error(scale)); + } + magnitude / divisor + }; + Ok(if sign == 1 { -magnitude } else { magnitude }) +} + +fn rounding_error(scale: u32) -> PyErr { + PyValueError::new_err(format!( + "decimal literal cannot be represented at scale {scale} without rounding" + )) +} + /// Operators recognized by the lightweight dict format but not translatable to a /// Rust [`Predicate`] for pushdown. const METHOD_NOT_SUPPORTED: &[&str] = &["like", "startsWith", "endsWith", "contains", "not"]; @@ -651,11 +885,410 @@ mod tests { } #[test] - fn timestamp_field_is_not_implemented() { + fn timestamp_field_rejects_non_datetime() { Python::attach(|py| { let v = 0i64.into_pyobject(py).unwrap(); let err = py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap_err(); - assert!(err.is_instance_of::(py)); + assert!(err.is_instance_of::(py)); + }); + } + + // ---- temporal literals ---- + + /// `datetime.date(2024, 1, 1)` epoch days (1970-01-01 = 0). + const D_2024_01_01: i32 = 19723; + + fn py_date<'py>(py: Python<'py>, y: i32, m: u8, d: u8) -> Bound<'py, PyAny> { + pyo3::types::PyDate::new(py, y, m, d).unwrap().into_any() + } + + fn py_time<'py>( + py: Python<'py>, + h: u8, + min: u8, + s: u8, + micro: u32, + tz: Option<&Bound<'py, pyo3::types::PyTzInfo>>, + ) -> Bound<'py, PyAny> { + pyo3::types::PyTime::new(py, h, min, s, micro, tz) + .unwrap() + .into_any() + } + + #[allow(clippy::too_many_arguments)] + fn py_datetime<'py>( + py: Python<'py>, + y: i32, + mo: u8, + d: u8, + h: u8, + mi: u8, + s: u8, + micro: u32, + tz: Option<&Bound<'py, pyo3::types::PyTzInfo>>, + ) -> Bound<'py, PyAny> { + pyo3::types::PyDateTime::new(py, y, mo, d, h, mi, s, micro, tz) + .unwrap() + .into_any() + } + + fn utc(py: Python<'_>) -> Bound<'_, pyo3::types::PyTzInfo> { + pyo3::types::PyTzInfo::utc(py).unwrap().to_owned() + } + + fn fixed_offset(py: Python<'_>, seconds: i32) -> Bound<'_, pyo3::types::PyTzInfo> { + let delta = pyo3::types::PyDelta::new(py, 0, seconds, 0, false).unwrap(); + pyo3::types::PyTzInfo::fixed_offset(py, &delta).unwrap() + } + + #[test] + fn date_field_accepts_date() { + Python::attach(|py| { + let v = py_date(py, 2024, 1, 1); + assert_eq!( + py_to_datum(&v, &DataType::Date(Default::default())).unwrap(), + Datum::Date(D_2024_01_01) + ); + }); + } + + #[test] + fn date_field_accepts_pre_epoch_date() { + Python::attach(|py| { + let v = py_date(py, 1969, 12, 31); + assert_eq!( + py_to_datum(&v, &DataType::Date(Default::default())).unwrap(), + Datum::Date(-1) + ); + }); + } + + #[test] + fn date_field_rejects_datetime() { + Python::attach(|py| { + // datetime is a date subclass; accepting it would silently drop the + // time-of-day part. + let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, None); + let err = py_to_datum(&v, &DataType::Date(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn date_field_rejects_str() { + Python::attach(|py| { + let v = "2024-01-01".into_pyobject(py).unwrap(); + let err = py_to_datum(v.as_any(), &DataType::Date(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn time_field_accepts_naive_time() { + Python::attach(|py| { + let v = py_time(py, 12, 34, 56, 789_000, None); + assert_eq!( + py_to_datum(&v, &DataType::Time(Default::default())).unwrap(), + Datum::Time(45_296_789) + ); + }); + } + + #[test] + fn time_field_rejects_sub_millisecond() { + Python::attach(|py| { + // Datum::Time carries millis-of-day; silently truncating micros + // would change equality semantics. + let v = py_time(py, 0, 0, 0, 500, None); + let err = py_to_datum(&v, &DataType::Time(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn time_field_rejects_aware_time() { + Python::attach(|py| { + let tz = utc(py); + let v = py_time(py, 1, 2, 3, 0, Some(&tz)); + let err = py_to_datum(&v, &DataType::Time(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn timestamp_field_accepts_naive_datetime() { + Python::attach(|py| { + let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 123_456, None); + assert_eq!( + py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap(), + Datum::Timestamp { + millis: 1_704_067_200_123, + nanos: 456_000, + } + ); + }); + } + + #[test] + fn timestamp_field_pre_epoch_uses_euclidean_split() { + Python::attach(|py| { + // 1969-12-31 23:59:59.999999 = -1 µs from epoch + // → floor millis -1, non-negative nanos remainder 999_000. + let v = py_datetime(py, 1969, 12, 31, 23, 59, 59, 999_999, None); + assert_eq!( + py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap(), + Datum::Timestamp { + millis: -1, + nanos: 999_000, + } + ); + }); + } + + #[test] + fn timestamp_field_rejects_aware_datetime() { + Python::attach(|py| { + // TIMESTAMP is timezone-less; parity with the DataFusion path which + // refuses zoned literals for it. + let tz = utc(py); + let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, Some(&tz)); + let err = py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn timestamp_field_rejects_date() { + Python::attach(|py| { + let v = py_date(py, 2024, 1, 1); + let err = py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn local_zoned_timestamp_field_accepts_aware_datetime() { + Python::attach(|py| { + // 2024-01-01T08:00:00+08:00 == 2024-01-01T00:00:00Z. + let tz = fixed_offset(py, 8 * 3600); + let v = py_datetime(py, 2024, 1, 1, 8, 0, 0, 0, Some(&tz)); + assert_eq!( + py_to_datum(&v, &DataType::LocalZonedTimestamp(Default::default())).unwrap(), + Datum::LocalZonedTimestamp { + millis: 1_704_067_200_000, + nanos: 0, + } + ); + }); + } + + #[test] + fn local_zoned_timestamp_field_rejects_naive_datetime() { + Python::attach(|py| { + let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, None); + let err = + py_to_datum(&v, &DataType::LocalZonedTimestamp(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + /// Build a datetime whose tzinfo subclass returns None from utcoffset() — + /// naive by Python's definition (docs: "aware if tzinfo is not None AND + /// utcoffset() does not return None") despite tzinfo being set. + fn pseudo_naive_datetime(py: Python<'_>) -> Bound<'_, PyAny> { + let ns = PyDict::new(py); + py.run( + c"import datetime +class FloatingTz(datetime.tzinfo): + def utcoffset(self, dt): return None + def dst(self, dt): return None + def tzname(self, dt): return 'FLOATING' +value = datetime.datetime(2024, 1, 1, tzinfo=FloatingTz())", + None, + Some(&ns), + ) + .unwrap(); + ns.get_item("value").unwrap().unwrap() + } + + #[test] + fn local_zoned_timestamp_field_rejects_pseudo_naive_tzinfo() { + Python::attach(|py| { + // tzinfo is set but utcoffset() is None: astimezone(utc) would + // interpret the value as process-local time, so the same literal + // would normalize differently per machine. Must be rejected. + let v = pseudo_naive_datetime(py); + let err = + py_to_datum(&v, &DataType::LocalZonedTimestamp(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn timestamp_field_rejects_pseudo_naive_tzinfo() { + Python::attach(|py| { + // Timestamp deliberately rejects ANY tzinfo, even one whose + // utcoffset() is None: stricter than Python's naive/aware + // definition, but explicit — the caller strips tzinfo to state + // wall-clock intent. + let v = pseudo_naive_datetime(py); + let err = py_to_datum(&v, &DataType::Timestamp(Default::default())).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + // ---- decimal literals ---- + + fn py_decimal<'py>(py: Python<'py>, repr: &str) -> Bound<'py, PyAny> { + py.import("decimal") + .unwrap() + .getattr("Decimal") + .unwrap() + .call1((repr,)) + .unwrap() + } + + fn decimal_type(precision: u32, scale: u32) -> DataType { + DataType::Decimal(paimon::spec::DecimalType::new(precision, scale).unwrap()) + } + + #[test] + fn decimal_field_accepts_exact_scale() { + Python::attach(|py| { + let v = py_decimal(py, "12.34"); + assert_eq!( + py_to_datum(&v, &decimal_type(10, 2)).unwrap(), + Datum::Decimal { + unscaled: 1234, + precision: 10, + scale: 2, + } + ); + }); + } + + #[test] + fn decimal_field_rescales_losslessly() { + Python::attach(|py| { + let v = py_decimal(py, "1.5"); + assert_eq!( + py_to_datum(&v, &decimal_type(10, 2)).unwrap(), + Datum::Decimal { + unscaled: 150, + precision: 10, + scale: 2, + } + ); + }); + } + + #[test] + fn decimal_field_accepts_scientific_notation() { + Python::attach(|py| { + let v = py_decimal(py, "1E+2"); + assert_eq!( + py_to_datum(&v, &decimal_type(10, 2)).unwrap(), + Datum::Decimal { + unscaled: 10_000, + precision: 10, + scale: 2, + } + ); + }); + } + + #[test] + fn decimal_field_negative_value() { + Python::attach(|py| { + let v = py_decimal(py, "-12.34"); + assert_eq!( + py_to_datum(&v, &decimal_type(10, 2)).unwrap(), + Datum::Decimal { + unscaled: -1234, + precision: 10, + scale: 2, + } + ); + }); + } + + #[test] + fn decimal_field_accepts_int_literal() { + Python::attach(|py| { + let v = 7i64.into_pyobject(py).unwrap(); + assert_eq!( + py_to_datum(&v, &decimal_type(10, 2)).unwrap(), + Datum::Decimal { + unscaled: 700, + precision: 10, + scale: 2, + } + ); + }); + } + + #[test] + fn decimal_field_rejects_rounding() { + Python::attach(|py| { + // 0.005 cannot be represented at scale 2 without rounding. + let v = py_decimal(py, "0.005"); + let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn decimal_field_rejects_precision_overflow() { + Python::attach(|py| { + let v = py_decimal(py, "123.45"); + let err = py_to_datum(&v, &decimal_type(4, 2)).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn decimal_field_rejects_non_finite() { + Python::attach(|py| { + for repr in ["NaN", "Infinity", "-Infinity"] { + let v = py_decimal(py, repr); + let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err(); + assert!(err.is_instance_of::(py), "{repr}"); + } + }); + } + + #[test] + fn decimal_field_rejects_float() { + Python::attach(|py| { + // Binary floats are not exact decimal values; requiring + // decimal.Decimal keeps the conversion lossless. + let v = 1.5f64.into_pyobject(py).unwrap(); + let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn decimal_field_rejects_bool() { + Python::attach(|py| { + let v = true.into_pyobject(py).unwrap(); + let err = py_to_datum(v.as_any(), &decimal_type(10, 2)).unwrap_err(); + assert!(err.is_instance_of::(py)); + }); + } + + #[test] + fn unsupported_field_type_still_not_implemented() { + Python::attach(|py| { + // Bytes/complex types remain out of scope for literal conversion. + let v = 0i64.into_pyobject(py).unwrap(); + for dt in [ + DataType::Binary(Default::default()), + DataType::Array(paimon::spec::ArrayType::new(DataType::Int(IntType::new()))), + ] { + let err = py_to_datum(&v, &dt).unwrap_err(); + assert!(err.is_instance_of::(py)); + } }); } diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py index 6d0f8f0a..1c40b256 100644 --- a/bindings/python/tests/test_read.py +++ b/bindings/python/tests/test_read.py @@ -196,17 +196,97 @@ def test_filter_unsupported_operator_precedes_shape_errors(): def test_filter_unsupported_type_raises(): - # Build a table with a timestamp column, filter on it -> NotImplementedError + # Binary columns have no literal conversion -> NotImplementedError with tempfile.TemporaryDirectory() as warehouse: ctx = SQLContext() ctx.register_catalog("paimon", {"warehouse": warehouse}) ctx.sql("CREATE SCHEMA paimon.tdb") - ctx.sql("CREATE TABLE paimon.tdb.tt (id INT, created_at TIMESTAMP)") - ctx.sql("INSERT INTO paimon.tdb.tt VALUES (1, TIMESTAMP '2024-01-01 00:00:00')") + ctx.sql("CREATE TABLE paimon.tdb.tt (id INT, payload BYTEA)") + ctx.sql("INSERT INTO paimon.tdb.tt VALUES (1, X'00')") table = PaimonCatalog({"warehouse": warehouse}).get_table("tdb.tt") with pytest.raises(NotImplementedError): table.new_read_builder().with_filter( - {"method": "equal", "field": "created_at", "literals": [0]}) + {"method": "equal", "field": "payload", "literals": [0]}) + + +def _make_temporal_table(warehouse): + ctx = SQLContext() + ctx.register_catalog("paimon", {"warehouse": warehouse}) + ctx.sql("CREATE SCHEMA paimon.tmp") + ctx.sql( + "CREATE TABLE paimon.tmp.tt (id INT, d DATE, ts TIMESTAMP, dec DECIMAL(10, 2))") + # Two separate INSERTs -> two files, so file stats can prune per-file. + ctx.sql( + "INSERT INTO paimon.tmp.tt VALUES " + "(1, DATE '2024-01-01', TIMESTAMP '2024-01-01 00:00:00', 12.34)") + ctx.sql( + "INSERT INTO paimon.tmp.tt VALUES " + "(2, DATE '2024-06-15', TIMESTAMP '2024-06-15 12:30:00', 56.78)") + return PaimonCatalog({"warehouse": warehouse}).get_table("tmp.tt") + + +def test_filter_date_literal_prunes_and_reads(): + import datetime + + with tempfile.TemporaryDirectory() as warehouse: + table = _make_temporal_table(warehouse) + b = table.new_read_builder().with_filter( + {"method": "equal", "field": "d", "literals": [datetime.date(2024, 1, 1)]}) + splits = b.new_scan().plan().splits() + assert len(splits) == 1 + t = pa.Table.from_batches(b.new_read().read(splits)) + assert t.column("id").to_pylist() == [1] + + +def test_filter_timestamp_literal_prunes_and_reads(): + import datetime + + with tempfile.TemporaryDirectory() as warehouse: + table = _make_temporal_table(warehouse) + b = table.new_read_builder().with_filter( + {"method": "greaterThan", "field": "ts", + "literals": [datetime.datetime(2024, 3, 1, 0, 0, 0)]}) + splits = b.new_scan().plan().splits() + assert len(splits) == 1 + t = pa.Table.from_batches(b.new_read().read(splits)) + assert t.column("id").to_pylist() == [2] + + +def test_filter_timestamp_rejects_aware_datetime(): + import datetime + + with tempfile.TemporaryDirectory() as warehouse: + table = _make_temporal_table(warehouse) + aware = datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc) + with pytest.raises(ValueError): + table.new_read_builder().with_filter( + {"method": "greaterThan", "field": "ts", "literals": [aware]}) + + +def test_filter_decimal_literal_prunes_and_reads(): + from decimal import Decimal + + with tempfile.TemporaryDirectory() as warehouse: + table = _make_temporal_table(warehouse) + b = table.new_read_builder().with_filter( + {"method": "equal", "field": "dec", "literals": [Decimal("12.34")]}) + splits = b.new_scan().plan().splits() + assert len(splits) == 1 + t = pa.Table.from_batches(b.new_read().read(splits)) + assert t.column("id").to_pylist() == [1] + + +def test_filter_decimal_rejects_float_and_rounding(): + from decimal import Decimal + + with tempfile.TemporaryDirectory() as warehouse: + table = _make_temporal_table(warehouse) + b = table.new_read_builder() + with pytest.raises(ValueError): + b.with_filter({"method": "equal", "field": "dec", "literals": [12.34]}) + with pytest.raises(ValueError): + b.with_filter( + {"method": "equal", "field": "dec", "literals": [Decimal("0.005")]}) def test_filter_unknown_field_raises():