Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions src/shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -2961,7 +2961,6 @@ def __init__(self, file: str | PathLike[Any] | ReadSeekableBinStream):

self.file.seek(0, 2) # offset=0, whence = 2 => EOF
self.file_size_B = self.file.tell()
self.file.seek(0)

@functools.cached_property
def file(self) -> ReadSeekableBinStream:
Expand All @@ -2978,7 +2977,6 @@ def _check_file_length(self, declared_file_length_B: int) -> None:
f"larger than actual file size: {self.file_size_B} (bytes)"
)

# TODO: If self.strict, raise instead?
msg = (
f"Declared file size in {file_name} header: {declared_file_length_B} "
f"smaller than actual file size: {self.file_size_B} "
Expand Down Expand Up @@ -3012,6 +3010,42 @@ def __init__(

self._dbfHeader()

self.file.seek(-1, 2) # one byte before EOF
self._last_byte = self.file.read(1)
last_byte_int = int.from_bytes(self._last_byte, byteorder="big")

declared_file_length_B = (
self._dbfHdrLength + self.numRecords * self._record_length + 1
)

if declared_file_length_B == self.file_size_B:
if last_byte_int != 0x1A:
msg = (
"Actual file size matches declared file size (header and records size + 1) "
f"but EOF byte: {last_byte_int:#x} is not 0x1a "
)
if self.strict:
raise dbfFileException(msg)
warnings.warn(msg)
elif declared_file_length_B - 1 == self.file_size_B:
# Missing EOF byte
msg = (
"File length is exactly the declared header and records size, "
"but misses the EOF byte. "
)
if last_byte_int == 0x1A:
msg += f"Last byte: {last_byte_int:#x} is not 0x1a (EOF byte special value) "
else:
msg += "Last byte is 0x1a, but that value could also be from the last record's data. "
msg += "Note: In previous versions of PyShp, Writers did not output a .dbf EOF byte "
if self.strict:
raise PossiblyCorruptFileHeader(msg)
msg += "so no warning will be given. Set strict=True to raise an Exception for this instead. "
logger.info(msg)

else:
self._check_file_length(declared_file_length_B)

def __len__(self) -> int:
"""Returns the number of records in the .dbf file."""
return self.numRecords
Expand All @@ -3027,10 +3061,6 @@ def _dbfHeader(self) -> None:
tuple[int, int, int], unpack("<xxxxLHH20x", self.file.read(32))
)

declared_file_length_B = (
self._dbfHdrLength + self.numRecords * self._record_length
)
self._check_file_length(declared_file_length_B)
# read fields
numFields = (self._dbfHdrLength - 33) // 32
for __ in range(numFields):
Expand Down Expand Up @@ -3064,7 +3094,7 @@ def fields(self) -> list[Field]:
return [Field("DeletionFlag", FieldType.C, 1, 0), *self.data_fields]

def _record_fmt(self, fields: Container[str] | None = None) -> str:
"""Calculates the struct format string for of a .dbf record. Optional 'fields' arg
"""Calculates the struct format string for a .dbf record. Optional 'fields' arg
specifies which fieldnames to unpack and which to ignore. Note that this
The DeletionFlag is always included at index 0 (to correctly unpack the dbf record).
The items in fields may appear in any order.
Expand Down Expand Up @@ -4451,6 +4481,11 @@ def _header(self) -> None:
# Terminator (0x0d from dbf spec https://en.wikipedia.org/wiki/.dbf#File_header)
f.write(b"\r")

def close(self) -> None:
# EOF byte (0x1a from dbf spec https://en.wikipedia.org/wiki/.dbf#File_format_of_Level_5_DOS_dBASE)
self.file.write(b"\x1a")
super().close()

def record(
self,
*recordList: RecordValue,
Expand Down
4 changes: 1 addition & 3 deletions tests/run_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
import shapefile

# For shapefiles from https://github.com/JamesParrott/PyShp_test_shapefile
DEFAULT_PYSHP_TEST_REPO = (
rf"{os.getenv('USERPROFILE')}\Coding\repos\PyShp_test_shapefile"
)
DEFAULT_PYSHP_TEST_REPO = Path(__file__).parent.parent.parent / "PyShp_test_shapefile"
PYSHP_TEST_REPO = Path(os.getenv("PYSHP_TEST_REPO", DEFAULT_PYSHP_TEST_REPO))
REPO_ROOT = Path(__file__).parent

Expand Down
2 changes: 1 addition & 1 deletion tests/test_shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -1336,7 +1336,7 @@ def test_reader_corrupt_files(tmp_path):
stopped = sf.dbf.tell()
sf.dbf.seek(0, 2)
end = sf.dbf.tell()
assert (end - stopped) == 5
assert (end - stopped) == 5 + 1 # length of byte string above, plus .dbf EOF byte
# assert that shapes are read without error
assert len(sf.shapes()) == 10
# assert that didn't read the extra junk data
Expand Down
Loading