diff --git a/ice-rest-catalog/pom.xml b/ice-rest-catalog/pom.xml index c606b98e..226e619c 100644 --- a/ice-rest-catalog/pom.xml +++ b/ice-rest-catalog/pom.xml @@ -510,7 +510,7 @@ com.google.guava guava - 33.5.0-jre + ${guava.version} org.checkerframework diff --git a/ice-rest-catalog/src/test/pyiceberg/uv.lock b/ice-rest-catalog/src/test/pyiceberg/uv.lock index d3edf4ff..6629e11e 100644 --- a/ice-rest-catalog/src/test/pyiceberg/uv.lock +++ b/ice-rest-catalog/src/test/pyiceberg/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" [[package]] @@ -35,11 +35,11 @@ wheels = [ [[package]] name = "certifi" -version = "2025.1.31" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577, upload-time = "2025-01-31T02:16:47.166Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -100,11 +100,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2025.3.0" +version = "2026.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491, upload-time = "2025-03-07T21:47:56.461Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615, upload-time = "2025-03-07T21:47:54.809Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" }, ] [[package]] @@ -171,11 +171,11 @@ dev = [{ name = "ruff", specifier = ">=0.11.2" }] [[package]] name = "idna" -version = "3.10" +version = "3.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] [[package]] @@ -337,11 +337,11 @@ wheels = [ [[package]] name = "pygments" -version = "2.19.1" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581, upload-time = "2025-01-06T17:26:30.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] @@ -478,7 +478,7 @@ wheels = [ [[package]] name = "requests" -version = "2.32.3" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -486,9 +486,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -570,11 +570,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index a003e1af..3a90371b 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -37,7 +37,8 @@ class ParquetConversions { private ParquetConversions() {} @SuppressWarnings("unchecked") - static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, Object value) { + static Literal fromParquetPrimitive( + Type type, PrimitiveType parquetType, Object value, boolean lowerBound) { switch (type.typeId()) { case BOOLEAN: return (Literal) Literal.of((Boolean) value); @@ -50,12 +51,8 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, case TIMESTAMP: // time & timestamp/timestamptz are stored in microseconds // https://iceberg.apache.org/spec/#parquet - var millis = - (parquetType.getLogicalTypeAnnotation() - instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation t - && t.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS); - var v = ((Number) value).longValue(); - return (Literal) Literal.of(millis ? v * 1000L : v); + return (Literal) + Literal.of(toMicroseconds(parquetType, ((Number) value).longValue(), lowerBound)); case FLOAT: return (Literal) Literal.of((Float) value); case DOUBLE: @@ -78,6 +75,35 @@ static Literal fromParquetPrimitive(Type type, PrimitiveType parquetType, } } + // Iceberg timestamp/timestamptz values are microseconds. NANOS footer stats are divided by + // 1000; a remainder widens the bound so the range still contains the original instant. + private static long toMicroseconds(PrimitiveType parquetType, long value, boolean lowerBound) { + if (!(parquetType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestamp)) { + return value; + } + return switch (timestamp.getUnit()) { + case MILLIS -> value * 1000L; + case NANOS -> nanosecondsToMicroseconds(value, lowerBound); + case MICROS -> value; + }; + } + + private static long nanosecondsToMicroseconds(long nanoseconds, boolean lowerBound) { + long microseconds = nanoseconds / 1000L; + if (nanoseconds % 1000L == 0) { + return microseconds; + } + if (lowerBound) { + if (nanoseconds < 0) { + microseconds--; + } + } else if (nanoseconds > 0) { + microseconds++; + } + return microseconds; + } + // Iceberg 1.9 introduced this API in upstream ParquetConversions and uses it from // ParquetMetrics. Provide a compatible shim here so callers compiled against 1.9 // continue to resolve against this shadowed class. The TIME/TIMESTAMP branch keeps diff --git a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java index 7d4f3290..456b6677 100644 --- a/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java +++ b/ice/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java @@ -145,11 +145,11 @@ public static Metrics footerMetrics( if (field != null && stats.hasNonNullValue() && shouldStoreBounds(column, fileSchema)) { Literal min = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMin()); + field.type(), column.getPrimitiveType(), stats.genericGetMin(), true); updateMin(lowerBounds, fieldId, field.type(), min, metricsMode); Literal max = ParquetConversions.fromParquetPrimitive( - field.type(), column.getPrimitiveType(), stats.genericGetMax()); + field.type(), column.getPrimitiveType(), stats.genericGetMax(), false); updateMax(upperBounds, fieldId, field.type(), max, metricsMode); } } diff --git a/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java new file mode 100644 index 00000000..815d1151 --- /dev/null +++ b/ice/src/test/java/org/apache/iceberg/parquet/ParquetConversionsTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package org.apache.iceberg.parquet; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.testng.annotations.Test; + +public class ParquetConversionsTest { + + private static final long NANOS = 1_786_467_726_576_815_622L; + private static final long MICROS = 1_786_467_726_576_815L; + + @Test + public void nanosBoundsWidenAroundTheInstant() { + PrimitiveType column = timestamp(LogicalTypeAnnotation.TimeUnit.NANOS); + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, column, NANOS, true)).isEqualTo(MICROS); + assertThat(bound(type, column, NANOS, false)).isEqualTo(MICROS + 1); + assertThat(bound(type, column, MICROS * 1000L, true)).isEqualTo(MICROS); + assertThat(bound(type, column, MICROS * 1000L, false)).isEqualTo(MICROS); + assertThat(bound(type, column, -1_500L, true)).isEqualTo(-2L); + assertThat(bound(type, column, -1_500L, false)).isEqualTo(-1L); + } + + @Test + public void millisScaleUpAndMicrosStay() { + Type type = Types.TimestampType.withZone(); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MILLIS), 5L, true)) + .isEqualTo(5_000L); + assertThat(bound(type, timestamp(LogicalTypeAnnotation.TimeUnit.MICROS), MICROS, false)) + .isEqualTo(MICROS); + } + + private static long bound(Type type, PrimitiveType column, long value, boolean lowerBound) { + return (Long) ParquetConversions.fromParquetPrimitive(type, column, value, lowerBound).value(); + } + + private static PrimitiveType timestamp(LogicalTypeAnnotation.TimeUnit unit) { + return org.apache.parquet.schema.Types.required(PrimitiveType.PrimitiveTypeName.INT64) + .as(LogicalTypeAnnotation.timestampType(true, unit)) + .named("ts"); + } +} diff --git a/pom.xml b/pom.xml index 2455bcf3..ea734178 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 1.15.2 2.31.13 4.7.6 - 2.18.8 + 2.18.11 2.0.17 1.5.18 1.3.6 @@ -34,12 +34,25 @@ 3.2.0 3.26.1 1.18.0 - 5.4.3 + 5.6.3 5.4.3 + 33.7.2-jre + + + com.google.guava + guava + ${guava.version} + + + + org.jspecify + jspecify + 1.0.0 + software.amazon.awssdk bom