diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b67fd7cc..c44d070e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Changelog version 1.10.0-dev ------------------ ++ Speed up ``read()`` without a size on ``igzip.open`` and ``IGzipFile`` files + that decompress to at most 64 KiB, on Linux with glibc. Reading many 3 KiB + gzip files by path is nearly 2 times faster. The gain applies in processes + that have not yet freed a memory block of 2 MiB or more. + Fix ``python -m isal.igzip`` replacing the input file with empty output and exiting with status 0 when the output file is the input file, as in ``-f -o data data``. It exits with an error now. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3ee12d83..ed592975 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2164,6 +2164,14 @@ GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) return NULL; } if (written_size < chunk_size) { + if (written_size <= chunk_size / 32) { + /* Shrinking an mmap()ed chunk keeps it mapped, so copy small + results and free the chunk. */ + PyObject *result = PyBytes_FromStringAndSize( + PyBytes_AS_STRING(first_chunk), written_size); + Py_DECREF(first_chunk); + return result; + } if (_PyBytes_Resize(&first_chunk, written_size) < 0) { return NULL; } diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 1c691c2b..6789a4ec 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -91,6 +91,38 @@ def test_igzip_reader_readall(): assert test.read(-1) == DATA +# Buffer size 1024: chunks of 4096 bytes, results up to 128 bytes copied. +@pytest.mark.parametrize("size", [0, 1, 127, 128, 129, 4095, 4096, 4097, + 3 * 4096 + 1]) +@pytest.mark.parametrize("members", [1, 2]) +def test_igzip_reader_readall_sizes(size, members): + data = os.urandom(size) + compressed = gzip.compress(data) * members + test = igzip._IGzipReader(io.BytesIO(compressed), 1024) + result = test.readall() + assert type(result) is bytes + assert result == data * members + assert test.tell() == size * members + assert test.readall() == b"" + + +@pytest.mark.parametrize("size", [100, 4096]) +def test_igzip_reader_readall_after_read(size): + data = os.urandom(size) + test = igzip._IGzipReader(io.BytesIO(gzip.compress(data)), 1024) + assert test.read(10) == data[:10] + assert test.readall() == data[10:] + assert test.tell() == size + + +# IGzipFile.read() copies results up to 64 KiB out of its 2 MiB chunk. +@pytest.mark.parametrize("size", [64 * 1024 - 1, 64 * 1024, 64 * 1024 + 1]) +def test_igzipfile_read_all_sizes(size): + data = os.urandom(size) + with igzip.IGzipFile(fileobj=io.BytesIO(gzip.compress(data))) as test: + assert test.read() == data + + def test_igzip_reader_read_zero(): data = io.BytesIO(COMPRESSED_DATA) test = igzip._IGzipReader(data)