diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index cd0d7cebcb..97663fcb8b 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -969,6 +969,16 @@ def _(_type: IcebergType, value: int) -> str: return datetime.to_human_timestamptz(value) +@_int_to_human_string.register(TimestampNanoType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamp_ns(value) + + +@_int_to_human_string.register(TimestamptzNanoType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamptz_ns(value) + + class UnknownTransform(Transform[S, T]): """A transform that represents when an unknown transform is provided. diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index e5fa2595fe..ad374bca03 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -218,6 +218,26 @@ def to_human_timestamp(timestamp_micros: int) -> str: return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat() +def to_human_timestamp_ns(timestamp_nanos: int) -> str: + """Convert a TimestampNanoType value to human string with nanosecond precision.""" + # Python datetime only supports microsecond precision, so render the + # microsecond timestamp with a fixed 6 fractional digits and append the + # remaining 3 sub-microsecond nanosecond digits for full 9-digit precision. + micros = nanos_to_micros(timestamp_nanos) + sub_micros = timestamp_nanos - micros * 1_000 + return (EPOCH_TIMESTAMP + timedelta(microseconds=micros)).isoformat(timespec="microseconds") + f"{sub_micros:03d}" + + +def to_human_timestamptz_ns(timestamp_nanos: int) -> str: + """Convert a TimestamptzNanoType value to human string with nanosecond precision.""" + micros = nanos_to_micros(timestamp_nanos) + sub_micros = timestamp_nanos - micros * 1_000 + iso = (EPOCH_TIMESTAMPTZ + timedelta(microseconds=micros)).isoformat(timespec="microseconds") + # Insert the sub-microsecond nanosecond digits before the "+00:00" zone offset. + timestamp_part, _, offset = iso.rpartition("+") + return f"{timestamp_part}{sub_micros:03d}+{offset}" + + def micros_to_hours(micros: int) -> int: """Convert a timestamp in microseconds to hours from 1970-01-01T00:00.""" return micros // 3_600_000_000 diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index b150fc2f67..3f6a9f951e 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -45,7 +45,9 @@ PrimitiveType, StringType, StructType, + TimestampNanoType, TimestampType, + TimestamptzNanoType, TimestamptzType, TimeType, UnknownType, @@ -172,6 +174,26 @@ def test_partition_spec_to_path() -> None: assert spec.partition_to_path(record, schema) == "my%23str%25bucket=my%2Bstr/other+str%2Bbucket=%28+%29/my%21int%3Abucket=10" +@pytest.mark.parametrize( + "field_type, expected_path", + [ + (TimestampType(), "ts=2025-02-23T20%3A21%3A44.375612"), + (TimestamptzType(), "ts=2025-02-23T20%3A21%3A44.375612%2B00%3A00"), + (TimestampNanoType(), "ts=2025-02-23T20%3A21%3A44.375612001"), + (TimestamptzNanoType(), "ts=2025-02-23T20%3A21%3A44.375612001%2B00%3A00"), + ], +) +def test_partition_spec_to_path_timestamp(field_type: PrimitiveType, expected_path: str) -> None: + schema = Schema(NestedField(field_id=1, name="ts", field_type=field_type, required=False)) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="ts")) + + # Nanosecond timestamps carry three extra sub-microsecond digits; the microsecond + # value 1740342104375612 is the same instant as 1740342104375612001 nanoseconds. + value = 1740342104375612001 if isinstance(field_type, (TimestampNanoType, TimestamptzNanoType)) else 1740342104375612 + + assert spec.partition_to_path(Record(value), schema) == expected_path + + def test_partition_spec_to_path_dropped_source_id() -> None: schema = Schema( NestedField(field_id=1, name="str", field_type=StringType(), required=False), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index d296fcdb21..82ff161689 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -416,6 +416,8 @@ def test_satisfies_order_of_method(transform: TimeTransform[Any], other_transfor (TimeType(), 36775038194, "10:12:55.038194"), (TimestamptzType(), 1512151975038194, "2017-12-01T18:12:55.038194+00:00"), (TimestampType(), 1512151975038194, "2017-12-01T18:12:55.038194"), + (TimestampNanoType(), 1740342104375612001, "2025-02-23T20:21:44.375612001"), + (TimestamptzNanoType(), 1740342104375612001, "2025-02-23T20:21:44.375612001+00:00"), (LongType(), -1234567890000, "-1234567890000"), (StringType(), "a/b/c=d", "a/b/c=d"), (DecimalType(9, 2), Decimal("-1.50"), "-1.50"), diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index e9529b6d3e..a2f439d602 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -29,6 +29,8 @@ time_to_nanos, timestamp_to_nanos, timestamptz_to_nanos, + to_human_timestamp_ns, + to_human_timestamptz_ns, ) timezones = [ @@ -154,3 +156,35 @@ def test_nanos_to_micros(nanos: int, micros: int) -> None: ) def test_nanos_to_hours(nanos: int, hours: int) -> None: assert hours == nanos_to_hours(nanos) + + +@pytest.mark.parametrize( + "nanos, human_str", + [ + (0, "1970-01-01T00:00:00.000000000"), + (1, "1970-01-01T00:00:00.000000001"), + (999, "1970-01-01T00:00:00.000000999"), + (1000, "1970-01-01T00:00:00.000001000"), + (1740342104375612001, "2025-02-23T20:21:44.375612001"), + (1510871468000001001, "2017-11-16T22:31:08.000001001"), + (1740342104000000001, "2025-02-23T20:21:44.000000001"), + ], +) +def test_to_human_timestamp_ns(nanos: int, human_str: str) -> None: + assert human_str == to_human_timestamp_ns(nanos) + + +@pytest.mark.parametrize( + "nanos, human_str", + [ + (0, "1970-01-01T00:00:00.000000000+00:00"), + (1, "1970-01-01T00:00:00.000000001+00:00"), + (999, "1970-01-01T00:00:00.000000999+00:00"), + (1000, "1970-01-01T00:00:00.000001000+00:00"), + (1740342104375612001, "2025-02-23T20:21:44.375612001+00:00"), + (1510871468000001001, "2017-11-16T22:31:08.000001001+00:00"), + (1740342104000000001, "2025-02-23T20:21:44.000000001+00:00"), + ], +) +def test_to_human_timestamptz_ns(nanos: int, human_str: str) -> None: + assert human_str == to_human_timestamptz_ns(nanos)