From c21210bc1ffcc9fe81f7f928efcc54e1836c5c03 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Fri, 6 Nov 2026 17:04:45 -0500 Subject: [PATCH] Keep the input as unconsumed_tail when decompress() reads none of it With a small max_length, isal_inflate often gives the output from data it already decoded into its own buffer, and reads no input. The decompress() method still copied all of its input to a new unconsumed_tail, so a loop that passes unconsumed_tail back in copied the whole rest of the compressed stream on most calls. When no input was read and the input is an exact bytes object that the buffer covers entirely, unconsumed_tail now keeps a reference to it. Other input, such as a bytearray, a memoryview or a bytes subclass, is still copied. Decompressing 4 MiB of Python source compressed at level 2 (1.2 MB) on Linux, passing unconsumed_tail back in, median of 5 rounds: with max_length=1024 it goes from 38 to 5.9 ms (6.4x) on Python 3.11 and from 39 to 5.9 ms (6.7x) on Python 3.14. 8192 is 1.5-1.7x faster and 16384 is 1.2x faster. 65536 and max_length=0 are unchanged. The only behavior change is identity: after a call that read no input, unconsumed_tail is the bytes object that was passed in. --- CHANGELOG.rst | 6 ++++ src/isal/isal_zlibmodule.c | 21 +++++++++++--- tests/test_compat.py | 57 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 481b7c5f..e2c8205a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,12 @@ Changelog version 1.10.0-dev ------------------ ++ Speed up ``decompress()`` of ``isal_zlib.decompressobj`` objects with a small + ``max_length``. When ISA-L gave the output from its own buffer without + reading any input, the whole input was still copied to ``unconsumed_tail``. + On Linux, decompressing 4 MiB of text 1 KiB at a time, passing + ``unconsumed_tail`` back in, is 5 times faster, and 8 KiB at a time is 1.5 + times faster. + Fix the gzip header that ``igzip_threaded.open`` writes when ``threads`` is not 0. It had the XFL and OS bytes swapped, so the ``file`` command reported the output as coming from VM/CMS or a FAT file system. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3ee12d83..fd379a38 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -983,10 +983,23 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) 1. Output limit was reached. Save leftover input in unconsumed_tail. 2. All input data was consumed. Clear unconsumed_tail. */ Py_ssize_t left_size = (uint8_t *)data->buf + data->len - self->zst.next_in; - PyObject *new_data = PyBytes_FromStringAndSize( - (char *)self->zst.next_in, left_size); - if (new_data == NULL) - return -1; + PyObject *new_data; + if (left_size == data->len && data->obj != NULL && + PyBytes_CheckExact(data->obj) && + data->buf == (void *)PyBytes_AS_STRING(data->obj) && + data->len == PyBytes_GET_SIZE(data->obj)) { + /* Nothing consumed: keep the input instead of copying it. The + buffer must cover all of obj, as an exporter may export only + part of a bytes object. */ + Py_INCREF(data->obj); + new_data = data->obj; + } + else { + new_data = PyBytes_FromStringAndSize( + (char *)self->zst.next_in, left_size); + if (new_data == NULL) + return -1; + } Py_SETREF(self->unconsumed_tail, new_data); } diff --git a/tests/test_compat.py b/tests/test_compat.py index 3f339851..3dfd6292 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -258,6 +258,63 @@ def test_decompressobj_unconsumed_tail(): assert len(output) == 2048 +class BytesSubclass(bytes): + pass + + +def memoryview_slice(data): + return memoryview(b"pad" + data)[3:] + + +@pytest.mark.parametrize(["input_type", "max_length", "wbits"], + list(itertools.product( + [bytes, BytesSubclass, bytearray, memoryview, + memoryview_slice], + [100, 1000, 8192], [-15, 15, 31]))) +def test_decompressobj_decompress_unconsumed_tail(input_type, max_length, + wbits): + data = DATA[:256 * 1024] + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + decompressor = isal_zlib.decompressobj(wbits=wbits) + unconsumed = compressed + b"unused" + result = [] + while not decompressor.eof: + chunk = input_type(unconsumed) + output = decompressor.decompress(chunk, max_length) + assert len(output) <= max_length + result.append(output) + unconsumed = decompressor.unconsumed_tail + assert type(unconsumed) is bytes + if isinstance(chunk, bytearray): + # The tail must be a copy, not the mutable input. + chunk[:] = bytes(len(chunk)) + result.append(decompressor.flush()) + assert b"".join(result) == data + assert decompressor.unconsumed_tail == b"" + assert decompressor.unused_data == b"unused" + + +@pytest.mark.skipif(not hasattr(sys, "getrefcount"), + reason="Needs reference counting") +def test_decompressobj_keeps_unconsumed_tail(): + data = DATA[:1024 * 1024] + compressed = zlib.compress(data) + decompressor = isal_zlib.decompressobj() + result = [decompressor.decompress(compressed, 1)] + tail = decompressor.unconsumed_tail + assert tail + refcount = sys.getrefcount(tail) + # Output comes from ISA-L's own buffer, so no input is consumed. + for _ in range(100): + result.append(decompressor.decompress(tail, 1)) + assert decompressor.unconsumed_tail is tail + assert sys.getrefcount(tail) == refcount + result.append(decompressor.flush()) + assert b"".join(result) == data + assert sys.getrefcount(tail) == refcount - 1 + + @pytest.mark.parametrize(["data_size", "level"], list(itertools.product(DATA_SIZES, range(4)))) def test_igzip_compress(data_size, level):