diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index a003e1af..3a90371b 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -37,7 +37,8 @@ class ParquetConversions { private ParquetConversions() {} @SuppressWarnings("unchecked") - static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, Object value) { + static Literal fromParquetPrimitive( + Type type, PrimitiveType parquetType, Object value, boolean lowerBound) { switch (type.typeId()) { case BOOLEAN: return (Literal) Literal.of((Boolean) value); @@ -50,12 +51,8 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, case TIMESTAMP: // time & timestamp/timestamptz are stored in microseconds // https://iceberg.apache.org/spec/#parquet - var millis = - (parquetType.getLogicalTypeAnnotation() - instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation t - && t.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS); - var v = ((Number) value).longValue(); - return (Literal) Literal.of(millis ? v * 1000L : v); + return (Literal) + Literal.of(toMicroseconds(parquetType, ((Number) value).longValue(), lowerBound)); case FLOAT: return (Literal) Literal.of((Float) value); case DOUBLE: @@ -78,6 +75,35 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, } } + // Iceberg timestamp/timestamptz values are microseconds. NANOS footer stats are divided by + // 1000; a remainder widens the bound so the range still contains the original instant. + private static long toMicroseconds(PrimitiveType parquetType, long value, boolean lowerBound) { + if (!(parquetType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestamp)) { + return value; + } + return switch (timestamp.getUnit()) { + case MILLIS -> value * 1000L; + case NANOS -> nanosecondsToMicroseconds(value, lowerBound); + case MICROS -> value; + }; + } + + private static long nanosecondsToMicroseconds(long nanoseconds, boolean lowerBound) { + long microseconds = nanoseconds / 1000L; + if (nanoseconds % 1000L == 0) { + return microseconds; + } + if (lowerBound) { + if (nanoseconds < 0) { + microseconds--; + } + } else if (nanoseconds > 0) { + microseconds++; + } + return microseconds; + } + // Iceberg 1.9 introduced this API in upstream ParquetConversions and uses it from // ParquetMetrics. Provide a compatible shim here so callers compiled against 1.9 // continue to resolve against this shadowed class. The TIME/TIMESTAMP branch keeps diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java index 7d4f3290..456b6677 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java @@ -145,11 +145,11 @@ public static Metrics footerMetrics( if (field != null && stats.hasNonNullValue() && shouldStoreBounds(column, fileSchema)) { Literal min = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMin()); + field.type(), column.getPrimitiveType(), stats.genericGetMin(), true); updateMin(lowerBounds, fieldId, field.type(), min, metricsMode); Literal max = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMax()); + field.type(), column.getPrimitiveType(), stats.genericGetMax(), false); updateMax(upperBounds, fieldId, field.type(), max, metricsMode); } } diff --git a/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java new file mode 100644 index 00000000..815d1151 --- /dev/null +++ b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package org.apache.iceberg.parquet; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.testng.annotations.Test; + +public class ParquetConversionsTest { + + private static final long NANOS = 1_786_467_726_576_815_622L; + private static final long MICROS = 1_786_467_726_576_815L; + + @Test + public void nanosBoundsWidenAroundTheInstant() { + PrimitiveType column = timestamp(LogicalTypeAnnotation.TimeUnit.NANOS); + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, column, NANOS, true)).isEqualTo(MICROS); + assertThat(bound(type, column, NANOS, false)).isEqualTo(MICROS + 1); + assertThat(bound(type, column, MICROS * 1000L, true)).isEqualTo(MICROS); + assertThat(bound(type, column, MICROS * 1000L, false)).isEqualTo(MICROS); + assertThat(bound(type, column, -1_500L, true)).isEqualTo(-2L); + assertThat(bound(type, column, -1_500L, false)).isEqualTo(-1L); + } + + @Test + public void millisScaleUpAndMicrosStay() { + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MILLIS), 5L, true)) + .isEqualTo(5_000L); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MICROS), MICROS, false)) + .isEqualTo(MICROS); + } + + private static long bound(Type type, PrimitiveType column, long value, boolean lowerBound) { + return (Long) ParquetConversions.fromParquetPrimitive(type, column, value, lowerBound).value(); + } + + private static PrimitiveType timestamp(LogicalTypeAnnotation.TimeUnit unit) { + return org.apache.parquet.schema.Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, unit)) + .named("ts"); + } +}