From 042d9607f65d04bfbe0b0daa3fa7555a3e27d3be Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:08:15 +0000 Subject: [PATCH] Write .dbf EOF bytes, & handle missing ones as a special case of the file length check Fix .dbf length calc Add byteorder kwarg to int.from_bytes Update hardcoded .dbf file length value in tests --- src/shapefile.py | 49 +++++++++++++++++++++++++++++++++++------ tests/run_benchmarks.py | 4 +--- tests/test_shapefile.py | 2 +- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/shapefile.py b/src/shapefile.py index cf0a9d8..e46dd91 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -2961,7 +2961,6 @@ def __init__(self, file: str | PathLike[Any] | ReadSeekableBinStream): self.file.seek(0, 2) # offset=0, whence = 2 => EOF self.file_size_B = self.file.tell() - self.file.seek(0) @functools.cached_property def file(self) -> ReadSeekableBinStream: @@ -2978,7 +2977,6 @@ def _check_file_length(self, declared_file_length_B: int) -> None: f"larger than actual file size: {self.file_size_B} (bytes)" ) - # TODO: If self.strict, raise instead? msg = ( f"Declared file size in {file_name} header: {declared_file_length_B} " f"smaller than actual file size: {self.file_size_B} " @@ -3012,6 +3010,42 @@ def __init__( self._dbfHeader() + self.file.seek(-1, 2) # one byte before EOF + self._last_byte = self.file.read(1) + last_byte_int = int.from_bytes(self._last_byte, byteorder="big") + + declared_file_length_B = ( + self._dbfHdrLength + self.numRecords * self._record_length + 1 + ) + + if declared_file_length_B == self.file_size_B: + if last_byte_int != 0x1A: + msg = ( + "Actual file size matches declared file size (header and records size + 1) " + f"but EOF byte: {last_byte_int:#x} is not 0x1a " + ) + if self.strict: + raise dbfFileException(msg) + warnings.warn(msg) + elif declared_file_length_B - 1 == self.file_size_B: + # Missing EOF byte + msg = ( + "File length is exactly the declared header and records size, " + "but misses the EOF byte. " + ) + if last_byte_int == 0x1A: + msg += f"Last byte: {last_byte_int:#x} is not 0x1a (EOF byte special value) " + else: + msg += "Last byte is 0x1a, but that value could also be from the last record's data. " + msg += "Note: In previous versions of PyShp, Writers did not output a .dbf EOF byte " + if self.strict: + raise PossiblyCorruptFileHeader(msg) + msg += "so no warning will be given. Set strict=True to raise an Exception for this instead. " + logger.info(msg) + + else: + self._check_file_length(declared_file_length_B) + def __len__(self) -> int: """Returns the number of records in the .dbf file.""" return self.numRecords @@ -3027,10 +3061,6 @@ def _dbfHeader(self) -> None: tuple[int, int, int], unpack(" list[Field]: return [Field("DeletionFlag", FieldType.C, 1, 0), *self.data_fields] def _record_fmt(self, fields: Container[str] | None = None) -> str: - """Calculates the struct format string for of a .dbf record. Optional 'fields' arg + """Calculates the struct format string for a .dbf record. Optional 'fields' arg specifies which fieldnames to unpack and which to ignore. Note that this The DeletionFlag is always included at index 0 (to correctly unpack the dbf record). The items in fields may appear in any order. @@ -4451,6 +4481,11 @@ def _header(self) -> None: # Terminator (0x0d from dbf spec https://en.wikipedia.org/wiki/.dbf#File_header) f.write(b"\r") + def close(self) -> None: + # EOF byte (0x1a from dbf spec https://en.wikipedia.org/wiki/.dbf#File_format_of_Level_5_DOS_dBASE) + self.file.write(b"\x1a") + super().close() + def record( self, *recordList: RecordValue, diff --git a/tests/run_benchmarks.py b/tests/run_benchmarks.py index 25f03eb..8888261 100644 --- a/tests/run_benchmarks.py +++ b/tests/run_benchmarks.py @@ -15,9 +15,7 @@ import shapefile # For shapefiles from https://github.com/JamesParrott/PyShp_test_shapefile -DEFAULT_PYSHP_TEST_REPO = ( - rf"{os.getenv('USERPROFILE')}\Coding\repos\PyShp_test_shapefile" -) +DEFAULT_PYSHP_TEST_REPO = Path(__file__).parent.parent.parent / "PyShp_test_shapefile" PYSHP_TEST_REPO = Path(os.getenv("PYSHP_TEST_REPO", DEFAULT_PYSHP_TEST_REPO)) REPO_ROOT = Path(__file__).parent diff --git a/tests/test_shapefile.py b/tests/test_shapefile.py index 33c7fb6..fc4d396 100644 --- a/tests/test_shapefile.py +++ b/tests/test_shapefile.py @@ -1336,7 +1336,7 @@ def test_reader_corrupt_files(tmp_path): stopped = sf.dbf.tell() sf.dbf.seek(0, 2) end = sf.dbf.tell() - assert (end - stopped) == 5 + assert (end - stopped) == 5 + 1 # length of byte string above, plus .dbf EOF byte # assert that shapes are read without error assert len(sf.shapes()) == 10 # assert that didn't read the extra junk data