From 4e37573243361406ab2feb42172ffb999a916478 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Fri, 6 Nov 2026 17:11:52 -0500 Subject: [PATCH] Open IGzipFile for reading without creating a zlib reader first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In read mode GzipFile.__init__ creates gzip's own _GzipReader, with a zlib decompressor, and an io.BufferedReader around it, and IGzipFile then replaced both with its own reader. When the mode is "r" or "rb", or neither a mode nor a fileobj is given, IGzipFile now does the read steps of GzipFile.__init__ itself and creates only its own reader. As GzipFile does on Python 3.14, it sets mode, fileobj and _buffer before it opens the file, and as on 3.12 and later, it closes the file it opened when a later step fails. IGzipFile(fileobj=f) without a mode still goes through GzipFile.__init__. On Linux, IGzipFile(fileobj=io.BytesIO(data), mode="rb") followed by read(100) and close() went from 4.2 to 2.1 µs on Python 3.11 and from 6.3 to 3.1 µs on 3.14 for a 100-byte stream, and from 6.2 to 3.4 µs and 6.8 to 4.3 µs for 4 KiB of text (medians of 7 interleaved rounds). That is about 34K fewer instructions per open on 3.11 and 41K on 3.14. Opening a small file by name and reading one line took 0.83x as long on both versions. In these modes compresslevel is now ignored, as in GzipFile, so a level that is not a number, or an out-of-range level without a mode, no longer raises TypeError. A file descriptor passed as the filename still raises TypeError, but the file is now closed at once, not at garbage collection with a ResourceWarning as before on Python 3.11. --- CHANGELOG.rst | 8 +++ src/isal/igzip.py | 27 ++++++++++ tests/test_igzip.py | 117 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 481b7c5f..055208c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,14 @@ Changelog version 1.10.0-dev ------------------ ++ Speed up opening an ``IGzipFile`` for reading with mode ``"r"`` or ``"rb"``, + which ``igzip.open`` uses by default, or by file name without a mode. It no + longer creates a ``zlib`` based reader first and throws it away. On Linux, + opening a small gzip stream in an ``io.BytesIO``, reading from it and + closing it takes 20-50% less time, and reading a line from a small file + opened by name 5-15% less. In these cases ``compresslevel`` is now + ignored, as in ``gzip.GzipFile``, so an invalid level no longer raises + ``TypeError``. + Fix the gzip header that ``igzip_threaded.open`` writes when ``threads`` is not 0. It had the XFL and OS bytes swapped, so the ``file`` command reported the output as coming from VM/CMS or a FAT file system. diff --git a/src/isal/igzip.py b/src/isal/igzip.py index b18af11c..794b2cf3 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -150,6 +150,33 @@ def __init__(self, filename=None, mode=None, to the last modification time field in the stream when compressing. If omitted or None, the current time is used. """ + if mode in ("r", "rb") or (mode is None and fileobj is None): + # GzipFile.__init__'s read branch, without its zlib reader. + # __del__ and close() need these even if opening fails. + self.mode = None + self.fileobj = None + self._buffer = None + try: + if fileobj is None: + fileobj = self.myfileobj = builtins.open(filename, "rb") + if filename is None: + filename = getattr(fileobj, "name", "") + if not isinstance(filename, (str, bytes)): + filename = "" + else: + filename = os.fspath(filename) + raw = _GzipReader(fileobj, READ_BUFFER_SIZE) + self._buffer = io.BufferedReader(raw) + except BaseException: + myfileobj = self.myfileobj + if myfileobj is not None: + self.myfileobj = None + myfileobj.close() + raise + self.mode = READ + self.name = filename + self.fileobj = fileobj + return if not (isal_zlib.ISAL_BEST_SPEED <= compresslevel <= isal_zlib.ISAL_BEST_COMPRESSION) and "r" not in mode: raise ValueError( diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 457f13f6..fa5d91d4 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -9,6 +9,7 @@ from CPython. Uses pytest which is easier to work with. Meant to complement the gzip module compliance tests. It should improve coverage as well.""" +import gc import gzip import io import itertools @@ -136,6 +137,122 @@ def test_igzipfile_read_truncated(): "reached") +@pytest.mark.parametrize("compressed", [b"", COMPRESSED_DATA, + COMPRESSED_DATA * 3], + ids=["empty", "one_member", "three_members"]) +@pytest.mark.parametrize("mode", ["r", "rb", None]) +def test_igzipfile_read_like_gzipfile(tmp_path, mode, compressed): + path = tmp_path / "test.gz" + path.write_bytes(compressed) + with gzip.GzipFile(path, mode) as gzip_file, \ + igzip.IGzipFile(path, mode) as igzip_file: + gzip_attributes = vars(gzip_file) + igzip_attributes = vars(igzip_file) + assert igzip_attributes.keys() == gzip_attributes.keys() + for name, value in gzip_attributes.items(): + assert type(igzip_attributes[name]) is type(value) + assert igzip_file.read() == gzip_file.read() + assert igzip_file.mtime == gzip_file.mtime + assert igzip_file.closed + assert igzip_file.myfileobj is None + with pytest.raises(ValueError, match="I/O operation on closed file"): + igzip_file.read() + + +@pytest.mark.parametrize(["mode", "use_fileobj"], [ + ("r", False), ("rb", False), (None, False), ("r", True), ("rb", True), +]) +def test_igzipfile_read_creates_no_gzip_reader(monkeypatch, tmp_path, mode, + use_fileobj): + def gzip_reader(*args, **kwargs): + raise AssertionError("gzip._GzipReader is replaced by IGzipFile") + + monkeypatch.setattr(gzip, "_GzipReader", gzip_reader) + path = tmp_path / "test.gz" + path.write_bytes(COMPRESSED_DATA) + if use_fileobj: + fileobj = io.BytesIO(COMPRESSED_DATA) + igzip_file = igzip.IGzipFile(fileobj=fileobj, mode=mode) + else: + igzip_file = igzip.IGzipFile(path, mode) + with igzip_file: + assert igzip_file.read() == DATA + + +@pytest.mark.parametrize("path_type", [str, os.fsencode, Path]) +@pytest.mark.parametrize("mode", ["r", "rb", None]) +def test_igzipfile_read_name_from_path(tmp_path, mode, path_type): + path = tmp_path / "test.gz" + path.write_bytes(COMPRESSED_DATA) + filename = path_type(path) + with igzip.IGzipFile(filename, mode) as igzip_file: + assert igzip_file.name == os.fspath(filename) + assert igzip_file.fileobj.name == os.fspath(filename) + assert igzip_file.read() == DATA + + +@pytest.mark.parametrize(["filename", "fileobj_name", "name"], [ + (None, None, ""), + (None, "file.gz", "file.gz"), + (None, b"file.gz", b"file.gz"), + (None, 3, ""), + ("other.gz", "file.gz", "other.gz"), + (b"other.gz", None, b"other.gz"), + (Path("other.gz"), None, "other.gz"), +]) +def test_igzipfile_read_name_with_fileobj(filename, fileobj_name, name): + fileobj = io.BytesIO(COMPRESSED_DATA) + if fileobj_name is not None: + fileobj.name = fileobj_name + with igzip.IGzipFile(filename, "rb", fileobj=fileobj) as igzip_file: + assert igzip_file.name == name + assert igzip_file.fileobj is fileobj + assert igzip_file.read() == DATA + + +@pytest.mark.parametrize("compresslevel", [None, "2", -1, 9]) +@pytest.mark.parametrize("mode", ["r", "rb", None]) +def test_igzipfile_read_ignores_compresslevel(tmp_path, mode, compresslevel): + path = tmp_path / "test.gz" + path.write_bytes(COMPRESSED_DATA) + with igzip.IGzipFile(path, mode, compresslevel) as igzip_file: + assert igzip_file.read() == DATA + + +@pytest.mark.parametrize("mode", ["r", "rb", None]) +def test_igzipfile_read_missing_file(monkeypatch, tmp_path, mode): + gc.collect() # Free earlier tests' objects first + unraisable = [] + monkeypatch.setattr(sys, "unraisablehook", unraisable.append) + with pytest.raises(FileNotFoundError): + igzip.IGzipFile(tmp_path / "missing.gz", mode) + gc.collect() + assert unraisable == [] + + +def test_igzipfile_read_closes_file_when_reader_fails(monkeypatch, tmp_path): + fileobjs = [] + + def failing_reader(fileobj, buffersize): + fileobjs.append(fileobj) + raise MemoryError() + + monkeypatch.setattr(igzip, "_GzipReader", failing_reader) + path = tmp_path / "test.gz" + path.write_bytes(COMPRESSED_DATA) + gc.collect() # Free earlier tests' objects first + unraisable = [] + monkeypatch.setattr(sys, "unraisablehook", unraisable.append) + with pytest.raises(MemoryError) as error: + igzip.IGzipFile(path, "rb") + # Closed now, not when the IGzipFile is collected. + assert len(fileobjs) == 1 + assert fileobjs[0].closed + del error + gc.collect() + assert unraisable == [] + + LINES_DATA = [ b"", b"no newline",