Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ice-rest-catalog/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>33.5.0-jre</version>
<version>${guava.version}</version>
<exclusions>
<exclusion>
<groupId>org.checkerframework</groupId>
Expand Down
38 changes: 19 additions & 19 deletions ice-rest-catalog/src/test/pyiceberg/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ class ParquetConversions {
private ParquetConversions() {}

@SuppressWarnings("unchecked")
static <T> Literal<T> fromParquetPrimitive(Type type, PrimitiveType parquetType, Object value) {
static <T> Literal<T> fromParquetPrimitive(
Type type, PrimitiveType parquetType, Object value, boolean lowerBound) {
switch (type.typeId()) {
case BOOLEAN:
return (Literal<T>) Literal.of((Boolean) value);
Expand All @@ -50,12 +51,8 @@ static <T> Literal<T> fromParquetPrimitive(Type type, PrimitiveType parquetType,
case TIMESTAMP:
// time & timestamp/timestamptz are stored in microseconds
// https://iceberg.apache.org/spec/#parquet
var millis =
(parquetType.getLogicalTypeAnnotation()
instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation t
&& t.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS);
var v = ((Number) value).longValue();
return (Literal<T>) Literal.of(millis ? v * 1000L : v);
return (Literal<T>)
Literal.of(toMicroseconds(parquetType, ((Number) value).longValue(), lowerBound));
case FLOAT:
return (Literal<T>) Literal.of((Float) value);
case DOUBLE:
Expand All @@ -78,6 +75,35 @@ static <T> Literal<T> fromParquetPrimitive(Type type, PrimitiveType parquetType,
}
}

// Iceberg timestamp/timestamptz values are microseconds. NANOS footer stats are divided by
// 1000; a remainder widens the bound so the range still contains the original instant.
private static long toMicroseconds(PrimitiveType parquetType, long value, boolean lowerBound) {
if (!(parquetType.getLogicalTypeAnnotation()
instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestamp)) {
return value;
}
return switch (timestamp.getUnit()) {
case MILLIS -> value * 1000L;
case NANOS -> nanosecondsToMicroseconds(value, lowerBound);
case MICROS -> value;
};
}

private static long nanosecondsToMicroseconds(long nanoseconds, boolean lowerBound) {
long microseconds = nanoseconds / 1000L;
if (nanoseconds % 1000L == 0) {
return microseconds;
}
if (lowerBound) {
if (nanoseconds < 0) {
microseconds--;
}
} else if (nanoseconds > 0) {
microseconds++;
}
return microseconds;
}

// Iceberg 1.9 introduced this API in upstream ParquetConversions and uses it from
// ParquetMetrics. Provide a compatible shim here so callers compiled against 1.9
// continue to resolve against this shadowed class. The TIME/TIMESTAMP branch keeps
Expand Down
4 changes: 2 additions & 2 deletions ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -145,11 +145,11 @@ public static Metrics footerMetrics(
if (field != null && stats.hasNonNullValue() && shouldStoreBounds(column, fileSchema)) {
Literal<?> min =
ParquetConversions.fromParquetPrimitive(
field.type(), column.getPrimitiveType(), stats.genericGetMin());
field.type(), column.getPrimitiveType(), stats.genericGetMin(), true);
updateMin(lowerBounds, fieldId, field.type(), min, metricsMode);
Literal<?> max =
ParquetConversions.fromParquetPrimitive(
field.type(), column.getPrimitiveType(), stats.genericGetMax());
field.type(), column.getPrimitiveType(), stats.genericGetMax(), false);
updateMax(upperBounds, fieldId, field.type(), max, metricsMode);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*/
package org.apache.iceberg.parquet;

import static org.assertj.core.api.Assertions.assertThat;

import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType;
import org.testng.annotations.Test;

public class ParquetConversionsTest {

private static final long NANOS = 1_786_467_726_576_815_622L;
private static final long MICROS = 1_786_467_726_576_815L;

@Test
public void nanosBoundsWidenAroundTheInstant() {
PrimitiveType column = timestamp(LogicalTypeAnnotation.TimeUnit.NANOS);
Type type = Types.TimestampType.withZone();
assertThat(bound(type, column, NANOS, true)).isEqualTo(MICROS);
assertThat(bound(type, column, NANOS, false)).isEqualTo(MICROS + 1);
assertThat(bound(type, column, MICROS * 1000L, true)).isEqualTo(MICROS);
assertThat(bound(type, column, MICROS * 1000L, false)).isEqualTo(MICROS);
assertThat(bound(type, column, -1_500L, true)).isEqualTo(-2L);
assertThat(bound(type, column, -1_500L, false)).isEqualTo(-1L);
}

@Test
public void millisScaleUpAndMicrosStay() {
Type type = Types.TimestampType.withZone();
assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MILLIS), 5L, true))
.isEqualTo(5_000L);
assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MICROS), MICROS, false))
.isEqualTo(MICROS);
}

private static long bound(Type type, PrimitiveType column, long value, boolean lowerBound) {
return (Long) ParquetConversions.fromParquetPrimitive(type, column, value, lowerBound).value();
}

private static PrimitiveType timestamp(LogicalTypeAnnotation.TimeUnit unit) {
return org.apache.parquet.schema.Types.required(PrimitiveType.PrimitiveTypeName.INT64)
.as(LogicalTypeAnnotation.timestampType(true, unit))
.named("ts");
}
}
17 changes: 15 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<parquet.version>1.15.2</parquet.version>
<aws.java.sdk.version>2.31.13</aws.java.sdk.version>
<picocli.version>4.7.6</picocli.version>
<jackson.version>2.18.8</jackson.version>
<jackson.version>2.18.11</jackson.version>
<slf4j.version>2.0.17</slf4j.version>
<logback.version>1.5.18</logback.version>
<prometheus.version>1.3.6</prometheus.version>
Expand All @@ -34,12 +34,25 @@
<caffeine.version>3.2.0</caffeine.version>
<jline.version>3.26.1</jline.version>
<commons.codec.version>1.18.0</commons.codec.version>
<httpclient5.version>5.4.3</httpclient5.version>
<httpclient5.version>5.6.3</httpclient5.version>
<httpcore5.version>5.4.3</httpcore5.version>
<guava.version>33.7.2-jre</guava.version>
</properties>

<dependencyManagement>
<dependencies>
<!-- pin transitive guava to a fixed release (Trivy: CVE-2026-102554) -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<!-- converge jspecify (enforcer DependencyConvergence): guava, caffeine, google-cloud-storage disagree -->
<dependency>
<groupId>org.jspecify</groupId>
<artifactId>jspecify</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
Expand Down
Loading