diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b67fd7cc..a4be7527 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,12 @@ Changelog version 1.10.0-dev ------------------ ++ Speed up ``decompress()`` of ``isal_zlib.decompressobj`` objects with a small + ``max_length``. When ISA-L gave the output from its own buffer without + reading any input, the whole input was still copied to ``unconsumed_tail``. + On Linux, decompressing 4 MiB of text 1 KiB at a time, passing + ``unconsumed_tail`` back in, is 5 times faster, and 8 KiB at a time is 1.5 + times faster. + Fix ``python -m isal.igzip`` replacing the input file with empty output and exiting with status 0 when the output file is the input file, as in ``-f -o data data``. It exits with an error now. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3ee12d83..fd379a38 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -983,10 +983,23 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) 1. Output limit was reached. Save leftover input in unconsumed_tail. 2. All input data was consumed. Clear unconsumed_tail. */ Py_ssize_t left_size = (uint8_t *)data->buf + data->len - self->zst.next_in; - PyObject *new_data = PyBytes_FromStringAndSize( - (char *)self->zst.next_in, left_size); - if (new_data == NULL) - return -1; + PyObject *new_data; + if (left_size == data->len && data->obj != NULL && + PyBytes_CheckExact(data->obj) && + data->buf == (void *)PyBytes_AS_STRING(data->obj) && + data->len == PyBytes_GET_SIZE(data->obj)) { + /* Nothing consumed: keep the input instead of copying it. The + buffer must cover all of obj, as an exporter may export only + part of a bytes object. */ + Py_INCREF(data->obj); + new_data = data->obj; + } + else { + new_data = PyBytes_FromStringAndSize( + (char *)self->zst.next_in, left_size); + if (new_data == NULL) + return -1; + } Py_SETREF(self->unconsumed_tail, new_data); } diff --git a/tests/test_compat.py b/tests/test_compat.py index 3f339851..3dfd6292 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -258,6 +258,63 @@ def test_decompressobj_unconsumed_tail(): assert len(output) == 2048 +class BytesSubclass(bytes): + pass + + +def memoryview_slice(data): + return memoryview(b"pad" + data)[3:] + + +@pytest.mark.parametrize(["input_type", "max_length", "wbits"], + list(itertools.product( + [bytes, BytesSubclass, bytearray, memoryview, + memoryview_slice], + [100, 1000, 8192], [-15, 15, 31]))) +def test_decompressobj_decompress_unconsumed_tail(input_type, max_length, + wbits): + data = DATA[:256 * 1024] + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + decompressor = isal_zlib.decompressobj(wbits=wbits) + unconsumed = compressed + b"unused" + result = [] + while not decompressor.eof: + chunk = input_type(unconsumed) + output = decompressor.decompress(chunk, max_length) + assert len(output) <= max_length + result.append(output) + unconsumed = decompressor.unconsumed_tail + assert type(unconsumed) is bytes + if isinstance(chunk, bytearray): + # The tail must be a copy, not the mutable input. + chunk[:] = bytes(len(chunk)) + result.append(decompressor.flush()) + assert b"".join(result) == data + assert decompressor.unconsumed_tail == b"" + assert decompressor.unused_data == b"unused" + + +@pytest.mark.skipif(not hasattr(sys, "getrefcount"), + reason="Needs reference counting") +def test_decompressobj_keeps_unconsumed_tail(): + data = DATA[:1024 * 1024] + compressed = zlib.compress(data) + decompressor = isal_zlib.decompressobj() + result = [decompressor.decompress(compressed, 1)] + tail = decompressor.unconsumed_tail + assert tail + refcount = sys.getrefcount(tail) + # Output comes from ISA-L's own buffer, so no input is consumed. + for _ in range(100): + result.append(decompressor.decompress(tail, 1)) + assert decompressor.unconsumed_tail is tail + assert sys.getrefcount(tail) == refcount + result.append(decompressor.flush()) + assert b"".join(result) == data + assert sys.getrefcount(tail) == refcount - 1 + + @pytest.mark.parametrize(["data_size", "level"], list(itertools.product(DATA_SIZES, range(4)))) def test_igzip_compress(data_size, level):