From 5f8f26fc4d8ffce0adc9a2967e481cf13d25d42b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 9 Nov 2026 14:28:19 +0100 Subject: [PATCH] fix: Convert Parquet NANOS footer stats to microseconds in manifest bounds `ice insert --no-copy` and `--force-no-copy` copy Parquet footer min/max into Iceberg manifest bounds. For a `timestamp`/`timestamptz` column the bounds must be microseconds, but NANOS statistics were copied unchanged, so a nanosecond file produced bounds about 1000x too large. Divide NANOS statistics by 1000. A remainder widens the lower bound down and the upper bound up, so the range still contains the original instant. MILLIS is still multiplied by 1000 and MICROS is unchanged. Fixes https://github.com/Altinity/ice/issues/227 Co-authored-by: Cursor --- .../iceberg/parquet/ParquetConversions.java | 40 +++++++++++--- .../apache/iceberg/parquet/ParquetUtil.java | 4 +- .../parquet/ParquetConversionsTest.java | 55 +++++++++++++++++++ 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index a003e1af..3a90371b 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -37,7 +37,8 @@ class ParquetConversions { private ParquetConversions() {} @SuppressWarnings("unchecked") - static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, Object value) { + static Literal fromParquetPrimitive( + Type type, PrimitiveType parquetType, Object value, boolean lowerBound) { switch (type.typeId()) { case BOOLEAN: return (Literal) Literal.of((Boolean) value); @@ -50,12 +51,8 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, case TIMESTAMP: // time & timestamp/timestamptz are stored in microseconds // https://iceberg.apache.org/spec/#parquet - var millis = - (parquetType.getLogicalTypeAnnotation() - instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation t - && t.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS); - var v = ((Number) value).longValue(); - return (Literal) Literal.of(millis ? v * 1000L : v); + return (Literal) + Literal.of(toMicroseconds(parquetType, ((Number) value).longValue(), lowerBound)); case FLOAT: return (Literal) Literal.of((Float) value); case DOUBLE: @@ -78,6 +75,35 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, } } + // Iceberg timestamp/timestamptz values are microseconds. NANOS footer stats are divided by + // 1000; a remainder widens the bound so the range still contains the original instant. + private static long toMicroseconds(PrimitiveType parquetType, long value, boolean lowerBound) { + if (!(parquetType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestamp)) { + return value; + } + return switch (timestamp.getUnit()) { + case MILLIS -> value * 1000L; + case NANOS -> nanosecondsToMicroseconds(value, lowerBound); + case MICROS -> value; + }; + } + + private static long nanosecondsToMicroseconds(long nanoseconds, boolean lowerBound) { + long microseconds = nanoseconds / 1000L; + if (nanoseconds % 1000L == 0) { + return microseconds; + } + if (lowerBound) { + if (nanoseconds < 0) { + microseconds--; + } + } else if (nanoseconds > 0) { + microseconds++; + } + return microseconds; + } + // Iceberg 1.9 introduced this API in upstream ParquetConversions and uses it from // ParquetMetrics. Provide a compatible shim here so callers compiled against 1.9 // continue to resolve against this shadowed class. The TIME/TIMESTAMP branch keeps diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java index 7d4f3290..456b6677 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java @@ -145,11 +145,11 @@ public static Metrics footerMetrics( if (field != null && stats.hasNonNullValue() && shouldStoreBounds(column, fileSchema)) { Literal min = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMin()); + field.type(), column.getPrimitiveType(), stats.genericGetMin(), true); updateMin(lowerBounds, fieldId, field.type(), min, metricsMode); Literal max = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMax()); + field.type(), column.getPrimitiveType(), stats.genericGetMax(), false); updateMax(upperBounds, fieldId, field.type(), max, metricsMode); } } diff --git a/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java new file mode 100644 index 00000000..815d1151 --- /dev/null +++ b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package org.apache.iceberg.parquet; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.testng.annotations.Test; + +public class ParquetConversionsTest { + + private static final long NANOS = 1_786_467_726_576_815_622L; + private static final long MICROS = 1_786_467_726_576_815L; + + @Test + public void nanosBoundsWidenAroundTheInstant() { + PrimitiveType column = timestamp(LogicalTypeAnnotation.TimeUnit.NANOS); + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, column, NANOS, true)).isEqualTo(MICROS); + assertThat(bound(type, column, NANOS, false)).isEqualTo(MICROS + 1); + assertThat(bound(type, column, MICROS * 1000L, true)).isEqualTo(MICROS); + assertThat(bound(type, column, MICROS * 1000L, false)).isEqualTo(MICROS); + assertThat(bound(type, column, -1_500L, true)).isEqualTo(-2L); + assertThat(bound(type, column, -1_500L, false)).isEqualTo(-1L); + } + + @Test + public void millisScaleUpAndMicrosStay() { + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MILLIS), 5L, true)) + .isEqualTo(5_000L); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MICROS), MICROS, false)) + .isEqualTo(MICROS); + } + + private static long bound(Type type, PrimitiveType column, long value, boolean lowerBound) { + return (Long) ParquetConversions.fromParquetPrimitive(type, column, value, lowerBound).value(); + } + + private static PrimitiveType timestamp(LogicalTypeAnnotation.TimeUnit unit) { + return org.apache.parquet.schema.Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, unit)) + .named("ts"); + } +}