diff --git a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md index 515a143488..4510f44fbe 100644 --- a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md +++ b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md @@ -146,6 +146,7 @@ Otherwise, libaec will need to be installed and reachable in your system library ### C-Blosc2 netCDF-Java `>=v5.10.0` supports blosc compression using JNA to call the C-Blosc2 library. +The `libblosc2-jna` module decodes data written with the HDF5 Blosc filter (id 32001) and the HDF5 Blosc2 filter (id 32026). To ease the use of this feature, we distribute a jar file containing the native libraries for the following platforms/architectures: |--- diff --git a/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java new file mode 100644 index 0000000000..fafbcba0c2 --- /dev/null +++ b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/Blosc2Filter.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package edu.ucar.unidata.compression.jna.libblosc2; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.filter.Filter; +import ucar.nc2.filter.FilterProvider; +import ucar.nc2.filter.Filters; + +/** + * Decoder for the HDF5 Blosc2 filter (id 32026). + *

+ * The filter parameters ({@code cd_values}) are set by the HDF5 plugin ({@code H5Zblosc2.c}): + * {@code [0]} filter revision, {@code [1]} Blosc2 version, {@code [2]} type size, {@code [3]} chunk size in + * bytes, {@code [4]} compression level, {@code [5]} shuffle, {@code [6]} codec, and for chunks of rank 2 or + * more {@code [7]} rank followed by the chunk shape. Only {@code [3]} is needed for decoding; shuffle and + * codec are recorded in each frame and handled by C-Blosc2. + */ +public class Blosc2Filter extends Filter { + private static final Logger logger = LoggerFactory.getLogger(Blosc2Filter.class); + private static final String name = "blosc2"; + private static final int id = 32026; + private static final int CD_CHUNK_BYTES = 3; + + private final Map properties; + private final int chunkBytes; + + public Blosc2Filter(Map properties) { + this.properties = new HashMap<>(properties); + Object data = this.properties.get(Filters.Keys.DATA); + this.chunkBytes = + (data instanceof int[] && ((int[]) data).length > CD_CHUNK_BYTES) ? ((int[]) data)[CD_CHUNK_BYTES] : -1; + logger.debug("Blosc2 properties = {}", this.properties); + } + + @Override + public String getName() { + return name; + } + + @Override + public int getId() { + return id; + } + + @Override + public byte[] encode(byte[] dataIn) { + throw new UnsupportedOperationException("Blosc2 encoding is not supported"); + } + + @Override + public byte[] decode(byte[] dataIn) throws IOException { + if (chunkBytes <= 0) { + Object data = properties.get(Filters.Keys.DATA); + throw new IOException("Blosc2 filter parameters do not contain the chunk size (cd_values[3]): " + + (data instanceof int[] ? Arrays.toString((int[]) data) : data)); + } + return LibBlosc2.decodeFrame(dataIn, chunkBytes); + } + + public Map properties() { + return new HashMap<>(properties); + } + + public static class Provider implements FilterProvider { + + @Override + public String getName() { + return name; + } + + @Override + public int getId() { + return id; + } + + @Override + public Filter create(Map properties) { + return new Blosc2Filter(properties); + } + } +} diff --git a/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/LibBlosc2.java b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/LibBlosc2.java index bbd6670a1d..32ca13a643 100644 --- a/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/LibBlosc2.java +++ b/native-compression/libblosc2-jna/src/main/java/edu/ucar/unidata/compression/jna/libblosc2/LibBlosc2.java @@ -17,7 +17,7 @@ /** * JNA access to C-Blosc2. Not a full implementation, just the functions * actually used for decoding (and testing). This is a partial transliteration - * of the C-Blosc2 library file include/blosc2.h. + * of the C-Blosc2 library files include/blosc2.h and include/b2nd.h. * * @author sarms * @since 5.10.0 @@ -116,6 +116,86 @@ public static byte[] decode(byte[] src) { return decompressed.getByteArray(0, nbytes.getValue().intValue()); } + /** + * Decode one chunk written by the HDF5 Blosc2 filter (id 32026, reference implementation + * {@code H5Zblosc2.c} in the HDF Group hdf5_plugins repository). + *

+ * Each HDF5 chunk is stored as one Blosc2 contiguous frame. Chunks of rank 2 or more carry a b2nd + * metalayer and are padded to a multiple of the block shape, so they are decoded with + * {@code b2nd_to_cbuffer}, which removes the padding. Rank 1 chunks hold a single Blosc2 chunk and + * are decoded with {@code blosc2_schunk_decompress_chunk}. This follows the decompression branch of + * {@code H5Zblosc2.c}. + * + * @param frame the chunk bytes as stored in the file (a Blosc2 frame) + * @param nbytes the size of the decoded HDF5 chunk in bytes ({@code cd_values[3]} of the filter); the + * uncompressed size in the frame header cannot be used because it may include padding + * @return the decoded chunk, exactly {@code nbytes} long + * @throws IOException if the frame is not a valid Blosc2 frame, is truncated or corrupt, or does not + * decode to {@code nbytes} bytes + */ + public static byte[] decodeFrame(byte[] frame, int nbytes) throws IOException { + if (frame == null || frame.length < FRAME_MAGIC_OFFSET + FRAME_MAGIC.length) { + throw new IOException("Blosc2 frame is too short: " + (frame == null ? "null" : frame.length + " bytes")); + } + for (int i = 0; i < FRAME_MAGIC.length; i++) { + if (frame[FRAME_MAGIC_OFFSET + i] != FRAME_MAGIC[i]) { + throw new IOException("Not a Blosc2 frame (missing \"b2frame\" magic)"); + } + } + if (nbytes <= 0) { + throw new IOException("Invalid decoded chunk size: " + nbytes); + } + init(); + // copy == false: the super-chunk refers to this memory, which must outlive it + try (Memory in = new Memory(frame.length); + Memory out = new Memory(nbytes); + Memory arrayRef = new Memory(Native.POINTER_SIZE)) { + in.write(0, frame, 0, frame.length); + Pointer schunk = blosc2_schunk_from_buffer(in, frame.length, (byte) 0); + if (schunk == null) { + throw new IOException( + "blosc2_schunk_from_buffer failed: corrupt or truncated frame (" + frame.length + " bytes)"); + } + try { + arrayRef.setPointer(0, null); + if (b2nd_from_schunk(schunk, arrayRef) >= 0) { + // rank >= 2: the b2nd array takes ownership of the super-chunk + Pointer array = arrayRef.getPointer(0); + schunk = null; + if (array == null) { + throw new IOException("b2nd_from_schunk did not return an array"); + } + try { + int rc = b2nd_to_cbuffer(array, out, nbytes); + if (rc < 0) { + throw new IOException("b2nd_to_cbuffer failed with error code " + rc); + } + } finally { + b2nd_free(array); + } + } else { + // rank 1: a single chunk + int written = blosc2_schunk_decompress_chunk(schunk, 0, out, nbytes); + if (written < 0) { + throw new IOException("blosc2_schunk_decompress_chunk failed with error code " + written); + } + if (written != nbytes) { + throw new IOException("Decoded " + written + " bytes, expected " + nbytes); + } + } + } finally { + if (schunk != null) { + blosc2_schunk_free(schunk); + } + } + return out.getByteArray(0, nbytes); + } + } + + // A Blosc2 contiguous frame starts with a msgpack header whose third byte begins "b2frame\0" + private static final byte[] FRAME_MAGIC = {'b', '2', 'f', 'r', 'a', 'm', 'e', 0}; + private static final int FRAME_MAGIC_OFFSET = 2; + // BLOSC_EXPORT void blosc2_init(void); static native void blosc2_init(); @@ -135,4 +215,23 @@ static native int blosc1_compress(int clevel, int doshuffle, NativeLong typesize static native void blosc1_cbuffer_sizes(byte[] compressedData, NativeLongByReference nbytes, NativeLongByReference cbytes, NativeLongByReference blocksize); + // BLOSC_EXPORT blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy); + static native Pointer blosc2_schunk_from_buffer(Pointer cframe, long len, byte copy); + + // BLOSC_EXPORT int blosc2_schunk_free(blosc2_schunk *schunk); + static native int blosc2_schunk_free(Pointer schunk); + + // BLOSC_EXPORT int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, void *dest, + // int32_t nbytes); + static native int blosc2_schunk_decompress_chunk(Pointer schunk, long nchunk, Pointer dest, int nbytes); + + // BLOSC_EXPORT int b2nd_from_schunk(blosc2_schunk *schunk, b2nd_array_t **array); + static native int b2nd_from_schunk(Pointer schunk, Pointer array); + + // BLOSC_EXPORT int b2nd_to_cbuffer(const b2nd_array_t *array, void *buffer, int64_t buffersize); + static native int b2nd_to_cbuffer(Pointer array, Pointer buffer, long buffersize); + + // BLOSC_EXPORT int b2nd_free(b2nd_array_t *array); + static native int b2nd_free(Pointer array); + } diff --git a/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider b/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider index 84d779258f..6c5474ac85 100644 --- a/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider +++ b/native-compression/libblosc2-jna/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider @@ -1 +1,2 @@ -edu.ucar.unidata.compression.jna.libblosc2.BloscFilter$Provider \ No newline at end of file +edu.ucar.unidata.compression.jna.libblosc2.BloscFilter$Provider +edu.ucar.unidata.compression.jna.libblosc2.Blosc2Filter$Provider diff --git a/native-compression/libblosc2-jna/src/test/data/README.md b/native-compression/libblosc2-jna/src/test/data/README.md new file mode 100644 index 0000000000..c67900ea60 --- /dev/null +++ b/native-compression/libblosc2-jna/src/test/data/README.md @@ -0,0 +1,34 @@ +# Blosc2 (HDF5 filter 32026) test data + +| File | Content | +|---|---| +| `blosc2_raw.nc` | netCDF-4, no filters (reference values) | +| `blosc2_32026.nc` | same data, every variable compressed with the HDF5 Blosc2 filter (id 32026) | + +Variables (dimensions `time = 12`, `y = 20`, `x = 30`): + +| Variable | Chunks | `_Filter` (user parameters) | Decoding path | +|---|---|---|---| +| `double time(time)` | 5 | `32026,0,0,0,0,3,0,5` (level 3, no shuffle, zstd) | rank 1: `blosc2_schunk_decompress_chunk` | +| `float temp(time, y, x)` | 5, 8, 16 | `32026,0,0,0,0,5,2,5` (level 5, bitshuffle, zstd) | rank 3: `b2nd_to_cbuffer` | +| `int count(y, x)` | 8, 16 | `32026,0,0,0,0,5,1,1` (level 5, byte shuffle, lz4) | rank 2: `b2nd_to_cbuffer` | + +The chunk sizes leave partial edge chunks in every dimension. + +Values: + +- `time[t] = 6 * t` +- `temp[t][j][i] = 250 + 0.5 t + 20 sin(0.3 j) cos(0.2 i) + 0.01 i` (written with 4 decimals) +- `count[j][i] = (31 j + 7 i) % 1000 - 500` + +Generated with netCDF-C 4.9.3, HDF5 1.14.6 and the HDF5 Blosc2 plugin (`H5Zblosc2.c` with C-Blosc2 2.23.1): +a CDL file with the values above was converted with `ncgen -k nc4`, once as is and once with +`_ChunkSizes` and `_Filter` attributes added. `ncdump` of both files gives identical data. + +Single chunks (for decoder unit tests), extracted from `blosc2_32026.nc` with `H5Dread_chunk` +(`.b2frame`, bytes as stored in the file) and `H5Dread` with the HDF5 Blosc2 plugin (`.expected`): + +| File | Chunk | Frame bytes | Decoded bytes | `cd_values` | +|---|---|---:|---:|---| +| `blosc2_temp_chunk0.*` | `temp` chunk 0 (rank 3, 5 x 8 x 16 floats) | 1203 | 2560 | `1,0,4,2560,5,2,5,3,5,8,16` | +| `blosc2_time_chunk0.*` | `time` chunk 0 (rank 1, 5 doubles) | 244 | 40 | `1,0,8,40,3,0,5` | diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc b/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc new file mode 100644 index 0000000000..9c95fecd49 Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_32026.nc differ diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc b/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc new file mode 100644 index 0000000000..90d2eea737 Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_raw.nc differ diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame new file mode 100644 index 0000000000..f7909cf1ea Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.b2frame differ diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected new file mode 100644 index 0000000000..e3b59d26c2 Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_temp_chunk0.expected differ diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame new file mode 100644 index 0000000000..03d98db343 Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.b2frame differ diff --git a/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected new file mode 100644 index 0000000000..e3dbe1b975 Binary files /dev/null and b/native-compression/libblosc2-jna/src/test/data/blosc2_time_chunk0.expected differ diff --git a/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java b/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java new file mode 100644 index 0000000000..35f3efc474 --- /dev/null +++ b/native-compression/libblosc2-jna/src/test/java/edu/ucar/unidata/compression/jna/libblosc2/TestBlosc2Filter.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package edu.ucar.unidata.compression.jna.libblosc2; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.nc2.Variable; +import ucar.nc2.filter.Filter; +import ucar.nc2.filter.Filters; +import ucar.nc2.filter.UnknownFilterException; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; + +public class TestBlosc2Filter { + private static final String DATA_DIR = "src/test/data/"; + private static final String RAW = DATA_DIR + "blosc2_raw.nc"; + private static final String COMPRESSED = DATA_DIR + "blosc2_32026.nc"; + private static final String[] VARIABLES = {"time", "temp", "count"}; + private static final int[] TEMP_CD_VALUES = {1, 0, 4, 2560, 5, 2, 5, 3, 5, 8, 16}; + private static final int[] TIME_CD_VALUES = {1, 0, 8, 40, 3, 0, 5}; + + @Test + public void testProviderIsRegistered() throws UnknownFilterException { + Map props = new HashMap<>(); + props.put(Filters.Keys.ID, (short) 32026); + props.put(Filters.Keys.DATA, TEMP_CD_VALUES); + Filter filter = Filters.getFilter(props); + assertThat(filter).isInstanceOf(Blosc2Filter.class); + assertThat(filter.getId()).isEqualTo(32026); + } + + @Test + public void testProviderByName() throws UnknownFilterException { + Map props = new HashMap<>(); + props.put(Filters.Keys.NAME, "blosc2"); + Filter filter = Filters.getFilter(props); + assertThat(filter).isInstanceOf(Blosc2Filter.class); + } + + @Test + public void testReadMatchesUncompressed() throws IOException { + try (NetcdfFile raw = NetcdfFiles.open(RAW); NetcdfFile compressed = NetcdfFiles.open(COMPRESSED)) { + for (String name : VARIABLES) { + Variable expected = raw.findVariable(name); + Variable actual = compressed.findVariable(name); + assertNotNull(expected); + assertNotNull(actual); + assertSameValues(name, expected.read(), actual.read()); + } + } + } + + @Test + public void testReadSubsetAcrossEdgeChunks() throws Exception { + // temp is chunked (5, 8, 16) over (12, 20, 30): this section touches interior and partial edge chunks + String section = "3:11,6:19,10:29"; + try (NetcdfFile raw = NetcdfFiles.open(RAW); NetcdfFile compressed = NetcdfFiles.open(COMPRESSED)) { + Array expected = raw.findVariable("temp").read(section); + Array actual = compressed.findVariable("temp").read(section); + assertSameValues("temp(" + section + ")", expected, actual); + } + } + + @Test + public void testMultithreadedRead() throws Exception { + Array expected; + try (NetcdfFile raw = NetcdfFiles.open(RAW)) { + expected = raw.findVariable("temp").read(); + } + ExecutorService executor = Executors.newFixedThreadPool(4); + try { + List> results = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + Callable task = () -> { + try (NetcdfFile compressed = NetcdfFiles.open(COMPRESSED)) { + return compressed.findVariable("temp").read(); + } + }; + results.add(executor.submit(task)); + } + for (Future result : results) { + assertSameValues("temp", expected, result.get()); + } + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testRejectsNonFrameInput() { + Blosc2Filter filter = newFilter(TEMP_CD_VALUES); + assertThrows(IOException.class, () -> filter.decode(new byte[] {1, 2, 3})); + assertThrows(IOException.class, () -> filter.decode(new byte[64])); + } + + @Test + public void testDecodeChunkRank3() throws IOException { + Blosc2Filter filter = newFilter(TEMP_CD_VALUES); + assertThat(filter.decode(readBytes("blosc2_temp_chunk0.b2frame"))) + .isEqualTo(readBytes("blosc2_temp_chunk0.expected")); + } + + @Test + public void testDecodeChunkRank1() throws IOException { + Blosc2Filter filter = newFilter(TIME_CD_VALUES); + assertThat(filter.decode(readBytes("blosc2_time_chunk0.b2frame"))) + .isEqualTo(readBytes("blosc2_time_chunk0.expected")); + } + + @Test + public void testRejectsTruncatedFrame() throws IOException { + Blosc2Filter filter = newFilter(TEMP_CD_VALUES); + byte[] frame = readBytes("blosc2_temp_chunk0.b2frame"); + for (int keep : new int[] {frame.length / 4, frame.length / 2, frame.length * 3 / 4, frame.length - 1}) { + byte[] truncated = Arrays.copyOf(frame, keep); + assertThrows(IOException.class, () -> filter.decode(truncated)); + } + } + + @Test + public void testRejectsWrongChunkSize() throws IOException { + int[] cdValues = TEMP_CD_VALUES.clone(); + cdValues[3] = 1280; // half of the real chunk size + Blosc2Filter filter = newFilter(cdValues); + assertThrows(IOException.class, () -> filter.decode(readBytes("blosc2_temp_chunk0.b2frame"))); + } + + private static byte[] readBytes(String name) throws IOException { + return Files.readAllBytes(Paths.get(DATA_DIR, name)); + } + + @Test + public void testRequiresChunkSizeParameter() { + Blosc2Filter filter = newFilter(new int[] {1, 0, 4}); + assertThrows(IOException.class, () -> filter.decode(new byte[64])); + } + + private static Blosc2Filter newFilter(int[] cdValues) { + Map props = new HashMap<>(); + props.put(Filters.Keys.ID, (short) 32026); + props.put(Filters.Keys.DATA, cdValues); + return new Blosc2Filter(props); + } + + private static void assertSameValues(String name, Array expected, Array actual) { + assertThat(actual.getElementType()).isEqualTo(expected.getElementType()); + assertThat(actual.getShape()).isEqualTo(expected.getShape()); + Object expectedValues = expected.copyTo1DJavaArray(); + Object actualValues = actual.copyTo1DJavaArray(); + if (expectedValues instanceof float[]) { + assertThat((float[]) actualValues).isEqualTo((float[]) expectedValues); + } else if (expectedValues instanceof double[]) { + assertThat((double[]) actualValues).isEqualTo((double[]) expectedValues); + } else if (expectedValues instanceof int[]) { + assertThat((int[]) actualValues).isEqualTo((int[]) expectedValues); + } else { + throw new AssertionError("unexpected type for " + name + ": " + expected.getElementType()); + } + } +}