diff --git a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md index 515a143488..4510f44fbe 100644 --- a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md +++ b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md @@ -146,6 +146,7 @@ Otherwise, libaec will need to be installed and reachable in your system library ### C-Blosc2 netCDF-Java `>=v5.10.0` supports blosc compression using JNA to call the C-Blosc2 library. +The `libblosc2-jna` module decodes data written with the HDF5 Blosc filter (id 32001) and the HDF5 Blosc2 filter (id 32026). To ease the use of this feature, we distribute a jar file containing the native libraries for the following platforms/architectures: |--- diff --git a/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java new file mode 100644 index 0000000000..fafbcba0c2 --- /dev/null +++ b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package edu.ucar.unidata.compression.jna.libblosc2; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.filter.Filter; +import ucar.nc2.filter.FilterProvider; +import ucar.nc2.filter.Filters; + +/** + * Decoder for the HDF5 Blosc2 filter (id 32026). + *
+ * The filter parameters ({@code cd_values}) are set by the HDF5 plugin ({@code H5Zblosc2.c}):
+ * {@code [0]} filter revision, {@code [1]} Blosc2 version, {@code [2]} type size, {@code [3]} chunk size in
+ * bytes, {@code [4]} compression level, {@code [5]} shuffle, {@code [6]} codec, and for chunks of rank 2 or
+ * more {@code [7]} rank followed by the chunk shape. Only {@code [3]} is needed for decoding; shuffle and
+ * codec are recorded in each frame and handled by C-Blosc2.
+ */
+public class Blosc2Filter extends Filter {
+ private static final Logger logger = LoggerFactory.getLogger(Blosc2Filter.class);
+ private static final String name = "blosc2";
+ private static final int id = 32026;
+ private static final int CD_CHUNK_BYTES = 3;
+
+ private final Map
+ * Each HDF5 chunk is stored as one Blosc2 contiguous frame. Chunks of rank 2 or more carry a b2nd
+ * metalayer and are padded to a multiple of the block shape, so they are decoded with
+ * {@code b2nd_to_cbuffer}, which removes the padding. Rank 1 chunks hold a single Blosc2 chunk and
+ * are decoded with {@code blosc2_schunk_decompress_chunk}. This follows the decompression branch of
+ * {@code H5Zblosc2.c}.
+ *
+ * @param frame the chunk bytes as stored in the file (a Blosc2 frame)
+ * @param nbytes the size of the decoded HDF5 chunk in bytes ({@code cd_values[3]} of the filter); the
+ * uncompressed size in the frame header cannot be used because it may include padding
+ * @return the decoded chunk, exactly {@code nbytes} long
+ * @throws IOException if the frame is not a valid Blosc2 frame, is truncated or corrupt, or does not
+ * decode to {@code nbytes} bytes
+ */
+ public static byte[] decodeFrame(byte[] frame, int nbytes) throws IOException {
+ if (frame == null || frame.length < FRAME_MAGIC_OFFSET + FRAME_MAGIC.length) {
+ throw new IOException("Blosc2 frame is too short: " + (frame == null ? "null" : frame.length + " bytes"));
+ }
+ for (int i = 0; i < FRAME_MAGIC.length; i++) {
+ if (frame[FRAME_MAGIC_OFFSET + i] != FRAME_MAGIC[i]) {
+ throw new IOException("Not a Blosc2 frame (missing \"b2frame\" magic)");
+ }
+ }
+ if (nbytes <= 0) {
+ throw new IOException("Invalid decoded chunk size: " + nbytes);
+ }
+ init();
+ // copy == false: the super-chunk refers to this memory, which must outlive it
+ try (Memory in = new Memory(frame.length);
+ Memory out = new Memory(nbytes);
+ Memory arrayRef = new Memory(Native.POINTER_SIZE)) {
+ in.write(0, frame, 0, frame.length);
+ Pointer schunk = blosc2_schunk_from_buffer(in, frame.length, (byte) 0);
+ if (schunk == null) {
+ throw new IOException(
+ "blosc2_schunk_from_buffer failed: corrupt or truncated frame (" + frame.length + " bytes)");
+ }
+ try {
+ arrayRef.setPointer(0, null);
+ if (b2nd_from_schunk(schunk, arrayRef) >= 0) {
+ // rank >= 2: the b2nd array takes ownership of the super-chunk
+ Pointer array = arrayRef.getPointer(0);
+ schunk = null;
+ if (array == null) {
+ throw new IOException("b2nd_from_schunk did not return an array");
+ }
+ try {
+ int rc = b2nd_to_cbuffer(array, out, nbytes);
+ if (rc < 0) {
+ throw new IOException("b2nd_to_cbuffer failed with error code " + rc);
+ }
+ } finally {
+ b2nd_free(array);
+ }
+ } else {
+ // rank 1: a single chunk
+ int written = blosc2_schunk_decompress_chunk(schunk, 0, out, nbytes);
+ if (written < 0) {
+ throw new IOException("blosc2_schunk_decompress_chunk failed with error code " + written);
+ }
+ if (written != nbytes) {
+ throw new IOException("Decoded " + written + " bytes, expected " + nbytes);
+ }
+ }
+ } finally {
+ if (schunk != null) {
+ blosc2_schunk_free(schunk);
+ }
+ }
+ return out.getByteArray(0, nbytes);
+ }
+ }
+
+ // A Blosc2 contiguous frame starts with a msgpack header whose third byte begins "b2frame\0"
+ private static final byte[] FRAME_MAGIC = {'b', '2', 'f', 'r', 'a', 'm', 'e', 0};
+ private static final int FRAME_MAGIC_OFFSET = 2;
+
// BLOSC_EXPORT void blosc2_init(void);
static native void blosc2_init();
@@ -135,4 +215,23 @@ static native int blosc1_compress(int clevel, int doshuffle, NativeLong typesize
static native void blosc1_cbuffer_sizes(byte[] compressedData, NativeLongByReference nbytes,
NativeLongByReference cbytes, NativeLongByReference blocksize);
+ // BLOSC_EXPORT blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy);
+ static native Pointer blosc2_schunk_from_buffer(Pointer cframe, long len, byte copy);
+
+ // BLOSC_EXPORT int blosc2_schunk_free(blosc2_schunk *schunk);
+ static native int blosc2_schunk_free(Pointer schunk);
+
+ // BLOSC_EXPORT int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, void *dest,
+ // int32_t nbytes);
+ static native int blosc2_schunk_decompress_chunk(Pointer schunk, long nchunk, Pointer dest, int nbytes);
+
+ // BLOSC_EXPORT int b2nd_from_schunk(blosc2_schunk *schunk, b2nd_array_t **array);
+ static native int b2nd_from_schunk(Pointer schunk, Pointer array);
+
+ // BLOSC_EXPORT int b2nd_to_cbuffer(const b2nd_array_t *array, void *buffer, int64_t buffersize);
+ static native int b2nd_to_cbuffer(Pointer array, Pointer buffer, long buffersize);
+
+ // BLOSC_EXPORT int b2nd_free(b2nd_array_t *array);
+ static native int b2nd_free(Pointer array);
+
}
diff --git a/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider b/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider
index 84d779258f..6c5474ac85 100644
--- a/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider
+++ b/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider
@@ -1 +1,2 @@
-edu.ucar.unidata.compression.jna.libblosc2.BloscFilter$Provider
\ No newline at end of file
+edu.ucar.unidata.compression.jna.libblosc2.BloscFilter$Provider
+edu.ucar.unidata.compression.jna.libblosc2.Blosc2Filter$Provider
diff --git a/native-compression/libblosc2-jna/src/test/data/README.md b/native-compression/libblosc2-jna/src/test/data/README.md
new file mode 100644
index 0000000000..c67900ea60
--- /dev/null
+++ b/native-compression/libblosc2-jna/src/test/data/README.md
@@ -0,0 +1,34 @@
+# Blosc2 (HDF5 filter 32026) test data
+
+| File | Content |
+|---|---|
+| `blosc2_raw.nc` | netCDF-4, no filters (reference values) |
+| `blosc2_32026.nc` | same data, every variable compressed with the HDF5 Blosc2 filter (id 32026) |
+
+Variables (dimensions `time = 12`, `y = 20`, `x = 30`):
+
+| Variable | Chunks | `_Filter` (user parameters) | Decoding path |
+|---|---|---|---|
+| `double time(time)` | 5 | `32026,0,0,0,0,3,0,5` (level 3, no shuffle, zstd) | rank 1: `blosc2_schunk_decompress_chunk` |
+| `float temp(time, y, x)` | 5, 8, 16 | `32026,0,0,0,0,5,2,5` (level 5, bitshuffle, zstd) | rank 3: `b2nd_to_cbuffer` |
+| `int count(y, x)` | 8, 16 | `32026,0,0,0,0,5,1,1` (level 5, byte shuffle, lz4) | rank 2: `b2nd_to_cbuffer` |
+
+The chunk sizes leave partial edge chunks in every dimension.
+
+Values:
+
+- `time[t] = 6 * t`
+- `temp[t][j][i] = 250 + 0.5 t + 20 sin(0.3 j) cos(0.2 i) + 0.01 i` (written with 4 decimals)
+- `count[j][i] = (31 j + 7 i) % 1000 - 500`
+
+Generated with netCDF-C 4.9.3, HDF5 1.14.6 and the HDF5 Blosc2 plugin (`H5Zblosc2.c` with C-Blosc2 2.23.1):
+a CDL file with the values above was converted with `ncgen -k nc4`, once as is and once with
+`_ChunkSizes` and `_Filter` attributes added. `ncdump` of both files gives identical data.
+
+Single chunks (for decoder unit tests), extracted from `blosc2_32026.nc` with `H5Dread_chunk`
+(`.b2frame`, bytes as stored in the file) and `H5Dread` with the HDF5 Blosc2 plugin (`.expected`):
+
+| File | Chunk | Frame bytes | Decoded bytes | `cd_values` |
+|---|---|---:|---:|---|
+| `blosc2_temp_chunk0.*` | `temp` chunk 0 (rank 3, 5 x 8 x 16 floats) | 1203 | 2560 | `1,0,4,2560,5,2,5,3,5,8,16` |
+| `blosc2_time_chunk0.*` | `time` chunk 0 (rank 1, 5 doubles) | 244 | 40 | `1,0,8,40,3,0,5` |
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc b/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc
new file mode 100644
index 0000000000..9c95fecd49
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc differ
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc b/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc
new file mode 100644
index 0000000000..90d2eea737
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc differ
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame
new file mode 100644
index 0000000000..f7909cf1ea
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame differ
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected
new file mode 100644
index 0000000000..e3b59d26c2
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected differ
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame
new file mode 100644
index 0000000000..03d98db343
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame differ
diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected
new file mode 100644
index 0000000000..e3dbe1b975
Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected differ
diff --git a/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java b/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java
new file mode 100644
index 0000000000..35f3efc474
--- /dev/null
+++ b/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
+package edu.ucar.unidata.compression.jna.libblosc2;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import org.junit.Test;
+import ucar.ma2.Array;
+import ucar.nc2.Variable;
+import ucar.nc2.filter.Filter;
+import ucar.nc2.filter.Filters;
+import ucar.nc2.filter.UnknownFilterException;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.NetcdfFiles;
+
+public class TestBlosc2Filter {
+ private static final String DATA_DIR = "src/test/data/";
+ private static final String RAW = DATA_DIR + "blosc2_raw.nc";
+ private static final String COMPRESSED = DATA_DIR + "blosc2_32026.nc";
+ private static final String[] VARIABLES = {"time", "temp", "count"};
+ private static final int[] TEMP_CD_VALUES = {1, 0, 4, 2560, 5, 2, 5, 3, 5, 8, 16};
+ private static final int[] TIME_CD_VALUES = {1, 0, 8, 40, 3, 0, 5};
+
+ @Test
+ public void testProviderIsRegistered() throws UnknownFilterException {
+ Map