From 44fba50442481f998d92ac9a574ae8257891d95b Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Fri, 6 Nov 2026 17:02:25 -0500 Subject: [PATCH] Copy small readall() results instead of shrinking the 2 MiB chunk IGzipFile.read() decompresses into a 2 MiB chunk (4 times the 512 KiB read buffer) and shrinks it to the result with realloc(). glibc serves a block this large with mmap(), and shrinking keeps it mapped, so every small read maps a new chunk, page-faults the pages it writes and unmaps it again. readall() now copies a result of at most 1/32 of the chunk (64 KiB for IGzipFile) into a new bytes object and frees the chunk whole. Freeing a 2 MiB mmap() block raises glibc's mmap threshold, so later chunks come from the heap without page faults. Fresh process, Linux/glibc in Docker, median of 5 interleaved runs, us per call, Python 3.11 and 3.14: - igzip.open(path).read() of 2000 3 KiB files: 33.2 -> 15.8 and 34.9 -> 16.6 - IGzipFile(fileobj=BytesIO).read() of 100 B: 19.9 -> 4.2 and 22.3 -> 5.0 - the same for 4 KiB: 28.4 -> 10.3 and 29.7 -> 11.2 - the same for 64 KiB: 113.4 -> 69.8 and 111.9 -> 69.2 Results over 64 KiB still shrink in place. Once a process has freed a 2 MiB block, glibc serves the chunk from the heap anyway; there the copy adds under 1% of instructions at 64 KiB and timings stay within noise. With mimalloc the change measured neutral. igzip.decompress() sizes its chunk from the input, so it rarely takes the copy path. --- CHANGELOG.rst | 4 ++++ src/isal/isal_zlibmodule.c | 8 ++++++++ tests/test_igzip.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 481b7c5f..91068c09 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Changelog version 1.10.0-dev ------------------ ++ Speed up ``read()`` without a size on ``igzip.open`` and ``IGzipFile`` files + that decompress to at most 64 KiB, on Linux with glibc. Reading many 3 KiB + gzip files by path is nearly 2 times faster. The gain applies in processes + that have not yet freed a memory block of 2 MiB or more. + Fix the gzip header that ``igzip_threaded.open`` writes when ``threads`` is not 0. It had the XFL and OS bytes swapped, so the ``file`` command reported the output as coming from VM/CMS or a FAT file system. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3ee12d83..ed592975 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2164,6 +2164,14 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) return NULL; } if (written_size < chunk_size) { + if (written_size <= chunk_size / 32) { + /* Shrinking an mmap()ed chunk keeps it mapped, so copy small + results and free the chunk. */ + PyObject *result = PyBytes_FromStringAndSize( + PyBytes_AS_STRING(first_chunk), written_size); + Py_DECREF(first_chunk); + return result; + } if (_PyBytes_Resize(&first_chunk, written_size) < 0) { return NULL; } diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 457f13f6..5449b408 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -91,6 +91,38 @@ def test_igzip_reader_readall(): assert test.read(-1) == DATA +# Buffer size 1024: chunks of 4096 bytes, results up to 128 bytes copied. +@pytest.mark.parametrize("size", [0, 1, 127, 128, 129, 4095, 4096, 4097, + 3 * 4096 + 1]) +@pytest.mark.parametrize("members", [1, 2]) +def test_igzip_reader_readall_sizes(size, members): + data = os.urandom(size) + compressed = gzip.compress(data) * members + test = igzip._IGzipReader(io.BytesIO(compressed), 1024) + result = test.readall() + assert type(result) is bytes + assert result == data * members + assert test.tell() == size * members + assert test.readall() == b"" + + +@pytest.mark.parametrize("size", [100, 4096]) +def test_igzip_reader_readall_after_read(size): + data = os.urandom(size) + test = igzip._IGzipReader(io.BytesIO(gzip.compress(data)), 1024) + assert test.read(10) == data[:10] + assert test.readall() == data[10:] + assert test.tell() == size + + +# IGzipFile.read() copies results up to 64 KiB out of its 2 MiB chunk. +@pytest.mark.parametrize("size", [64 * 1024 - 1, 64 * 1024, 64 * 1024 + 1]) +def test_igzipfile_read_all_sizes(size): + data = os.urandom(size) + with igzip.IGzipFile(fileobj=io.BytesIO(gzip.compress(data))) as test: + assert test.read() == data + + def test_igzip_reader_read_zero(): data = io.BytesIO(COMPRESSED_DATA) test = igzip._IGzipReader(data)