From f01c76248a62f030a9dbba5e5e6417a33b53a968 Mon Sep 17 00:00:00 2001 From: Marlon Date: Fri, 13 Nov 2026 02:23:54 +0100 Subject: [PATCH 1/3] feat!: harden the readers against corrupted data Every read is now checked against the data before anything is read or allocated. Before, corrupted lengths could read past the end of a buffer, return stale pooled memory, or allocate gigabytes before failing, and the readers threw different exceptions for the same input. - Running out of data throws EndOfStreamException, an impossible length in the data InvalidDataException, an invalid argument ArgumentOutOfRangeException. A failed read leaves Position unchanged, apart from a length prefix that was already read. - Positions and seek targets are validated, so they can no longer be negative or wrap around. - ReadArray with a getter is limited by MaxArrayAllocationSize (default 256 MiB), since its length cannot be checked against the data. - GenericFileReader reads through a fixed-size cache; larger reads go straight to the file, and a file that grows or shrinks after opening is handled. - GenericStreamReader puts the stream back where a failed read started. - GenericFileReader and GenericStreamReader take a leaveOpen argument. - After Dispose, the buffer, file and stream readers throw ObjectDisposedException for reads and position access. BREAKING CHANGE: raw reads require T : unmanaged. The readers over memory return ReadOnlySpan/ReadOnlyMemory instead of writable views, and FStringMemory holds ReadOnlyMemory. Corrupted data throws the exceptions listed above instead of IndexOutOfRangeException or OverflowException. Dispose is implemented once in GenericReaderBase; subclasses override Dispose(bool). Co-Authored-By: Claude Opus 5.5 --- src/GenericReader.Tests/Tests.cs | 2 +- src/GenericReader/FStringMemory.cs | 7 +- src/GenericReader/GenericBufferReader.cs | 313 +++++++++---------- src/GenericReader/GenericFileReader.cs | 337 +++++++++++++++------ src/GenericReader/GenericReaderBase.cs | 129 +++++--- src/GenericReader/GenericSpanReader.cs | 367 ++++++++++------------- src/GenericReader/GenericStreamReader.cs | 243 ++++++++------- src/GenericReader/IGenericReader.cs | 28 +- src/GenericReader/ReaderHelpers.cs | 96 ++++++ src/GenericReader/TempBuffer.cs | 34 +++ src/GenericReader/ThrowHelper.cs | 37 +++ 11 files changed, 982 insertions(+), 611 deletions(-) create mode 100644 src/GenericReader/ReaderHelpers.cs create mode 100644 src/GenericReader/TempBuffer.cs create mode 100644 src/GenericReader/ThrowHelper.cs diff --git a/src/GenericReader.Tests/Tests.cs b/src/GenericReader.Tests/Tests.cs index dae51bd..472ca43 100644 --- a/src/GenericReader.Tests/Tests.cs +++ b/src/GenericReader.Tests/Tests.cs @@ -196,7 +196,7 @@ public BinaryDataValues() } private void Add() - where TValue : struct, IEquatable + where TValue : unmanaged, IEquatable { Unsafe.SkipInit(out TValue value); Helpers.Fill(ref value); diff --git a/src/GenericReader/FStringMemory.cs b/src/GenericReader/FStringMemory.cs index 740f841..8d3936b 100644 --- a/src/GenericReader/FStringMemory.cs +++ b/src/GenericReader/FStringMemory.cs @@ -2,16 +2,17 @@ namespace GenericReader; +// The characters of an FString, without its terminator, as a view of the reader's memory. public readonly struct FStringMemory { - public Memory Memory { get; } + public ReadOnlyMemory Memory { get; } public bool IsUnicode { get; } - public Span GetSpan() => Memory.Span; + public ReadOnlySpan GetSpan() => Memory.Span; public bool IsEmpty() => Memory.IsEmpty; public Encoding GetEncoding() => IsUnicode ? Encoding.Unicode : Encoding.UTF8; - public FStringMemory(Memory memory, bool isUnicode) + public FStringMemory(ReadOnlyMemory memory, bool isUnicode) { Memory = memory; IsUnicode = isUnicode; diff --git a/src/GenericReader/GenericBufferReader.cs b/src/GenericReader/GenericBufferReader.cs index 2782b0d..2b01b0c 100644 --- a/src/GenericReader/GenericBufferReader.cs +++ b/src/GenericReader/GenericBufferReader.cs @@ -8,29 +8,39 @@ namespace GenericReader; public class GenericBufferReader : GenericReaderBase { - private Memory _memory; + private readonly ReadOnlyMemory _memory; + private int _position; - public GenericBufferReader(byte[] buffer, int start, int length) : this(new Memory(buffer, start, length)) { } - public GenericBufferReader(byte[] buffer) : this(new Memory(buffer)) { } - public GenericBufferReader(ReadOnlyMemory memory) : this(MemoryMarshal.AsMemory(memory)) { } - public GenericBufferReader(Memory memory) + public GenericBufferReader(byte[] buffer, int start, int length) : this(new ReadOnlyMemory(buffer, start, length)) { } + public GenericBufferReader(byte[] buffer) : this(new ReadOnlyMemory(buffer)) { } + public GenericBufferReader(ReadOnlyMemory memory) { _memory = memory; - LengthLong = Length = memory.Length; } - internal void SetMemoryAndPosition(byte[] buffer, int position) + public override int Position { - Position = position; - _memory = buffer; + get + { + ThrowIfDisposed(); + return _position; + } + set + { + ThrowIfDisposed(); + ArgumentOutOfRangeException.ThrowIfNegative(value); + _position = value; + } } - public override int Position { get; set; } - public override long PositionLong { get => Position; - set => Position = int.CreateChecked(value); + set + { + ThrowIfDisposed(); + _position = ReaderHelpers.ToInt32Position(value); + } } public override TPosition GetPosition() @@ -40,130 +50,131 @@ public override TPosition GetPosition() public override void SetPosition(TPosition position) { - Position = int.CreateChecked(position); + ThrowIfDisposed(); + _position = (int)ReaderHelpers.ToPosition(position, int.MaxValue); } - public override int Length { get; } - public override long LengthLong { get; } + public override int Length => _memory.Length; + public override long LengthLong => _memory.Length; public override TLength GetLength() { - return TLength.CreateChecked(Length); + return TLength.CreateChecked(_memory.Length); } public override int Seek(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { SeekVoid(offset, origin); - return Position; + return _position; } public override long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { SeekVoid(offset, origin); - return Position; + return _position; } public override void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - Position = origin switch - { - SeekOrigin.Begin => int.CreateChecked(offset), - SeekOrigin.Current => Position + int.CreateChecked(offset), - SeekOrigin.End => Length + int.CreateChecked(offset), - _ => throw new ArgumentOutOfRangeException(nameof(origin), origin, null) - }; + ThrowIfDisposed(); + _position = (int)ReaderHelpers.GetSeekTarget(offset, origin, _position, _memory.Length, int.MaxValue); } - public override T Read() where T : struct + // Throws unless byteCount (not negative) bytes are left from Position. Position is never negative, so the + // subtraction cannot overflow; it is negative when Position lies past the end. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureAvailable(long byteCount) { - T result = Unsafe.ReadUnaligned(ref _memory.Span[Position]); - int size = Unsafe.SizeOf(); - Position += size; - return result; + ThrowIfDisposed(); + if (byteCount > _memory.Length - _position) + ThrowHelper.ThrowEndOfStream(); + } + + // The next byteCount (not negative) bytes, without moving Position. The two Peek overloads are the only places + // that create spans over the data, so no span is created without a bounds check. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ReadOnlySpan Peek(int byteCount) + { + ThrowIfDisposed(); + ReadOnlySpan span = _memory.Span; + // both are at most int.MaxValue, so the unsigned sum cannot wrap around + if ((uint)_position + (uint)byteCount > (uint)span.Length) + ThrowHelper.ThrowEndOfStream(); + + return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), (nuint)(uint)_position), byteCount); } - public override void Read(Span dest) where T : struct + // Peek for a computed byte count such as length * SizeOf, which may not fit in an int. + private ReadOnlySpan Peek(long byteCount) + { + EnsureAvailable(byteCount); + // at most the length of the data now, so the cast cannot truncate + return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.Add(ref MemoryMarshal.GetReference(_memory.Span), (nuint)(uint)_position), (int)byteCount); + } + + public override T Read() { int size = Unsafe.SizeOf(); - Span span = MemoryMarshal.CreateSpan(ref Unsafe.As(ref dest[0]), dest.Length * size); - _memory.Span.Slice(Position, span.Length).CopyTo(span); - Position += span.Length; + T result = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(Peek(size))); + _position += size; + return result; } - public override string ReadString(int length, Encoding enc) + public override void Read(Span dest) { - return ReadString(length, enc, false); + // 64-bit: a destination of more than int.MaxValue bytes cannot fit in the data, but must not overflow either + ReadOnlySpan source = Peek((long)dest.Length * Unsafe.SizeOf()); + source.CopyTo(MemoryMarshal.AsBytes(dest)); + _position += source.Length; } - internal string ReadString(int length, Encoding enc, bool trimNull) + public override string ReadString(int length, Encoding enc) { - Span span = _memory.Span.Slice(Position, length); - if (trimNull) - span = span.TrimEnd(byte.MinValue); - string result = enc.GetString(span); - Position += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentNullException.ThrowIfNull(enc); + string result = enc.GetString(Peek(length)); + _position += length; return result; } public override string ReadFString() { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization int length = Read(); if (length == 0) return string.Empty; - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); - - int pLength = length * -sizeof(char); - Span span = _memory.Span.Slice(Position, pLength - sizeof(char)); - string result = Encoding.Unicode.GetString(span); - Position += pLength; - return result; - } - else - { - Span span = _memory.Span.Slice(Position, length).TrimEnd(byte.MinValue); - string result = Encoding.UTF8.GetString(span); - Position += length; - return result; - } + int byteCount = ReaderHelpers.GetFStringByteCount(length); + string result = ReaderHelpers.DecodeFString(Peek(byteCount), length); + _position += byteCount; + return result; } public FStringMemory ReadFStringMemory() { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization int length = Read(); if (length == 0) return default; - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); - - int pLength = length * -sizeof(char); - Memory memory = _memory.Slice(Position, pLength - sizeof(char)); - Position += pLength; - return new FStringMemory(memory, true); - } - else - { - Memory memory = _memory.Slice(Position, length).TrimEnd(byte.MinValue); - Position += length; - return new FStringMemory(memory, false); - } + int byteCount = ReaderHelpers.GetFStringByteCount(length); + EnsureAvailable(byteCount); + ReadOnlyMemory memory = _memory.Slice(_position, byteCount); + _position += byteCount; + + // without the terminator + return length < 0 + ? new FStringMemory(memory[..^sizeof(char)], true) + : new FStringMemory(memory.TrimEnd(byte.MinValue), false); } public FStringMemory[] ReadFStringMemoryArray(int length) { - FStringMemory[] result = GC.AllocateUninitializedArray(length); + ArgumentOutOfRangeException.ThrowIfNegative(length); + // every FString takes at least 4 bytes (its length), so a larger count is rejected before allocating + EnsureAvailable((long)length * sizeof(int)); + if (length == 0) + return []; + + FStringMemory[] result = new FStringMemory[length]; for (int i = 0; i < length; i++) result[i] = ReadFStringMemory(); @@ -171,130 +182,128 @@ public FStringMemory[] ReadFStringMemoryArray(int length) return result; } - public FStringMemory[] ReadFStringMemoryArray() - { - int length = Read(); - return ReadFStringMemoryArray(length); - } + public FStringMemory[] ReadFStringMemoryArray() => ReadFStringMemoryArray(ReadLength()); - public override T[] ReadArray(int length) where T : struct + public override T[] ReadArray(int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that the byte count cannot wrap around, and checked before allocating + ReadOnlySpan source = Peek((long)length * Unsafe.SizeOf()); if (length == 0) return []; - int size = length * Unsafe.SizeOf(); T[] result = GC.AllocateUninitializedArray(length); - Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref result[0]), ref _memory.Span[Position], (uint)size); - Position += size; + source.CopyTo(MemoryMarshal.AsBytes(result.AsSpan())); + _position += source.Length; return result; } public GenericBufferReader Slice(int start, bool sliceAtPosition = false) { - int sliceStart = sliceAtPosition ? start + Position : start; - return new GenericBufferReader(_memory.Slice(sliceStart)); + int sliceStart = sliceAtPosition ? start + _position : start; + return new GenericBufferReader(_memory.Slice(sliceStart)) { MaxArrayAllocationSize = MaxArrayAllocationSize }; } public GenericBufferReader Slice(int start, int length, bool sliceAtPosition = false) { - int sliceStart = sliceAtPosition ? start + Position : start; - return new GenericBufferReader(_memory.Slice(sliceStart, length)); + int sliceStart = sliceAtPosition ? start + _position : start; + return new GenericBufferReader(_memory.Slice(sliceStart, length)) { MaxArrayAllocationSize = MaxArrayAllocationSize }; } public static GenericBufferReader LoadFromFile(SafeFileHandle handle) { - long length = RandomAccess.GetLength(handle); - byte[] buffer = new byte[length]; - RandomAccess.Read(handle, buffer, 0); + ArgumentNullException.ThrowIfNull(handle); + byte[] buffer = ReaderHelpers.AllocateBuffer(RandomAccess.GetLength(handle)); + + // RandomAccess.Read may return fewer bytes than requested + for (int totalRead = 0; totalRead < buffer.Length;) + { + int bytesRead = RandomAccess.Read(handle, buffer.AsSpan(totalRead), totalRead); + if (bytesRead == 0) + ThrowHelper.ThrowEndOfStream(); // the file got shorter + totalRead += bytesRead; + } + return new GenericBufferReader(buffer); } public static async ValueTask LoadFromFileAsync(SafeFileHandle handle, CancellationToken cancellationToken = default) { - long length = RandomAccess.GetLength(handle); - byte[] buffer = new byte[length]; - var memory = new Memory(buffer); - await RandomAccess.ReadAsync(handle, memory, 0, cancellationToken).ConfigureAwait(false); - return new GenericBufferReader(memory); + ArgumentNullException.ThrowIfNull(handle); + byte[] buffer = ReaderHelpers.AllocateBuffer(RandomAccess.GetLength(handle)); + + // RandomAccess.ReadAsync may return fewer bytes than requested + for (int totalRead = 0; totalRead < buffer.Length;) + { + int bytesRead = await RandomAccess.ReadAsync(handle, buffer.AsMemory(totalRead), totalRead, cancellationToken) + .ConfigureAwait(false); + if (bytesRead == 0) + ThrowHelper.ThrowEndOfStream(); // the file got shorter + totalRead += bytesRead; + } + + return new GenericBufferReader(buffer); } + // Loads the rest of the stream, from its current position on. public static GenericBufferReader LoadFromStream(Stream stream) { - byte[] buffer = new byte[stream.Length]; + ArgumentNullException.ThrowIfNull(stream); + byte[] buffer = ReaderHelpers.AllocateBuffer(stream.Length - stream.Position); stream.ReadExactly(buffer); return new GenericBufferReader(buffer); } + // Loads the rest of the stream, from its current position on. public static async ValueTask LoadFromStreamAsync(Stream stream, CancellationToken cancellationToken = default) { - byte[] buffer = new byte[stream.Length]; - var memory = new Memory(buffer); - await stream.ReadExactlyAsync(memory, cancellationToken).ConfigureAwait(false); - return new GenericBufferReader(memory); + ArgumentNullException.ThrowIfNull(stream); + byte[] buffer = ReaderHelpers.AllocateBuffer(stream.Length - stream.Position); + await stream.ReadExactlyAsync(buffer, cancellationToken).ConfigureAwait(false); + return new GenericBufferReader(buffer); } - public Memory AsMemory(bool sliceAtPosition = false) + public ReadOnlyMemory AsMemory(bool sliceAtPosition = false) { - return sliceAtPosition ? _memory.Slice(Position) : _memory; + return sliceAtPosition ? _memory.Slice(_position) : _memory; } - public Span AsSpan(bool sliceAtPosition = false) + public ReadOnlySpan AsSpan(bool sliceAtPosition = false) { - return sliceAtPosition ? _memory.Span.Slice(Position) : _memory.Span; + return sliceAtPosition ? _memory.Span.Slice(_position) : _memory.Span; } - public Memory ReadMemory() - { - int length = Read(); - return ReadMemory(length); - } + public ReadOnlyMemory ReadMemory() => ReadMemory(ReadLength()); - public Memory ReadMemory(int length) + public ReadOnlyMemory ReadMemory(int length) { - Memory result = _memory.Slice(Position, length); - Position += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + EnsureAvailable(length); + ReadOnlyMemory result = _memory.Slice(_position, length); + _position += length; return result; } - public Span ReadSpan() - { - int length = Read(); - return ReadSpan(length); - } + public ReadOnlySpan ReadSpan() => ReadSpan(ReadLength()); - public Span ReadSpan(int length) + public ReadOnlySpan ReadSpan(int length) { - Span result = _memory.Span.Slice(Position, length); - Position += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + ReadOnlySpan result = Peek(length); + _position += length; return result; } - public Span ReadSpan() where T : struct - { - int length = Read(); - return ReadSpan(length); - } - - public Span ReadSpan(int length) where T : struct - { - int size = length * Unsafe.SizeOf(); - Span memorySpan = _memory.Span.Slice(Position, size); - ref T reference = ref Unsafe.As(ref MemoryMarshal.GetReference(memorySpan)); - Span resultSpan = MemoryMarshal.CreateSpan(ref reference, length); - Position += size; - return resultSpan; - } - - protected virtual void Dispose(bool disposing) - { - if (disposing) { } - } + public ReadOnlySpan ReadSpan() where T : unmanaged => ReadSpan(ReadLength()); - public override void Dispose() + public ReadOnlySpan ReadSpan(int length) where T : unmanaged { - Dispose(true); - GC.SuppressFinalize(this); + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that a wrapped byte count cannot pass the check while the span keeps the full length + ReadOnlySpan source = Peek((long)length * Unsafe.SizeOf()); + _position += source.Length; + return MemoryMarshal.Cast(source); } } diff --git a/src/GenericReader/GenericFileReader.cs b/src/GenericReader/GenericFileReader.cs index 9df3a6a..ff7a1ff 100644 --- a/src/GenericReader/GenericFileReader.cs +++ b/src/GenericReader/GenericFileReader.cs @@ -1,67 +1,83 @@ using System.Buffers; using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using Microsoft.Win32.SafeHandles; namespace GenericReader; +// Reads a file through a cache of bufferSize bytes. Reads larger than the cache go straight to the file, so that +// a single large read neither grows the cache nor makes every later cache miss read that much. public class GenericFileReader : GenericReaderBase { private const int DefaultBufferSize = 4096; private readonly SafeFileHandle _handle; - private readonly GenericBufferReader _bufferReader; + private readonly bool _leaveOpen; + private readonly int _bufferSize; - private byte[] _buffer; - private long _bufferPosition; - private long _bufferEndPosition; - private int _bufferSize; - private int _bufferSizeToAlloc; + private byte[]? _buffer; // rented from BufferPool on the first cache miss, returned by Dispose + private long _bufferOffset; // the file offset of _buffer[0] + private int _bufferLength; // how many bytes of _buffer hold file data + private long _position; + + // Where the cache comes from; the tests swap it for a pool that counts what is rented and returned. + internal ArrayPool BufferPool { get; init; } = ArrayPool.Shared; public GenericFileReader(string filePath, int bufferSize = DefaultBufferSize) - : this(File.OpenHandle(filePath, options: FileOptions.RandomAccess), bufferSize) { } - public GenericFileReader(SafeFileHandle handle, int bufferSize = DefaultBufferSize) + : this(OpenHandle(filePath, bufferSize), bufferSize) { } + + // Unless leaveOpen, the reader owns the handle: it disposes it, also when this constructor fails after the + // arguments were validated. + public GenericFileReader(SafeFileHandle handle, int bufferSize = DefaultBufferSize, bool leaveOpen = false) { - _buffer = []; + ArgumentNullException.ThrowIfNull(handle); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bufferSize); _handle = handle; - long fileLength = RandomAccess.GetLength(_handle); - LengthLong = fileLength; - Length = unchecked((int)fileLength); - - _bufferSizeToAlloc = bufferSize; - _bufferPosition = _bufferEndPosition = -1; - _bufferReader = new GenericBufferReader(Memory.Empty); - } + _leaveOpen = leaveOpen; + _bufferSize = bufferSize; - private void EnsureBufferAllocated(int size) - { - if (size > _bufferSize || PositionLong < _bufferPosition || PositionLong + size > _bufferEndPosition) + try { - _bufferSizeToAlloc = Math.Max(_bufferSizeToAlloc, size); - if (_bufferPosition != -1) - ArrayPool.Shared.Return(_buffer); - _buffer = ArrayPool.Shared.Rent(_bufferSizeToAlloc); - int bytesRead = RandomAccess.Read(_handle, _buffer, PositionLong); - _bufferPosition = PositionLong; - _bufferSize = bytesRead; - _bufferEndPosition = PositionLong + bytesRead; - _bufferReader.SetMemoryAndPosition(_buffer, (int)(PositionLong - _bufferPosition)); + LengthLong = RandomAccess.GetLength(handle); } - else + catch { - _bufferReader.Position = (int)(PositionLong - _bufferPosition); + if (!leaveOpen) + handle.Dispose(); + throw; } } + // validates bufferSize first, so that an invalid one does not leave an open handle behind + private static SafeFileHandle OpenHandle(string filePath, int bufferSize) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bufferSize); + return File.OpenHandle(filePath, options: FileOptions.RandomAccess); + } + public override int Position { get => int.CreateChecked(PositionLong); set => PositionLong = value; } - public override long PositionLong { get; set; } + public override long PositionLong + { + get + { + ThrowIfDisposed(); + return _position; + } + set + { + ThrowIfDisposed(); + ArgumentOutOfRangeException.ThrowIfNegative(value); + _position = value; + } + } public override TPosition GetPosition() { @@ -70,10 +86,11 @@ public override TPosition GetPosition() public override void SetPosition(TPosition position) { - PositionLong = long.CreateChecked(position); + ThrowIfDisposed(); + _position = ReaderHelpers.ToPosition(position, long.MaxValue); } - public override int Length { get; } + public override int Length => int.CreateChecked(LengthLong); public override long LengthLong { get; } public override TLength GetLength() @@ -83,129 +100,255 @@ public override TLength GetLength() public override int Seek(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - return unchecked((int)SeekLong(offset, origin)); + // checked against int.MaxValue before Position moves + ThrowIfDisposed(); + _position = ReaderHelpers.GetSeekTarget(offset, origin, _position, LengthLong, int.MaxValue); + return (int)_position; } public override long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { SeekVoid(offset, origin); - return PositionLong; + return _position; } public override void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - PositionLong = origin switch + ThrowIfDisposed(); + _position = ReaderHelpers.GetSeekTarget(offset, origin, _position, LengthLong, long.MaxValue); + } + + // Throws unless byteCount (not negative) bytes are left from Position. Position is never negative, so the + // subtraction cannot overflow; it is negative when Position lies past the end. + private void EnsureAvailable(long byteCount) + { + ThrowIfDisposed(); + if (byteCount > LengthLong - _position) + ThrowHelper.ThrowEndOfStream(); + } + + // The next byteCount bytes (not negative, at most _bufferSize) from the cache, without moving Position. After + // Dispose the cache is empty, so that every read misses it and FillBuffer throws. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ReadOnlySpan PeekBuffered(int byteCount) + { + // Unsigned, so that this cannot overflow: both offsets are never negative, and a Position before the cache + // turns into a huge offset. + ulong offset = (ulong)(_position - _bufferOffset); + if (offset > (ulong)_bufferLength || (ulong)byteCount > (ulong)_bufferLength - offset) { - SeekOrigin.Begin => long.CreateChecked(offset), - SeekOrigin.Current => PositionLong + long.CreateChecked(offset), - SeekOrigin.End => LengthLong + long.CreateChecked(offset), - _ => throw new ArgumentOutOfRangeException(nameof(origin), origin, null) - }; + FillBuffer(byteCount); + offset = 0; + } + + return _buffer.AsSpan((int)offset, byteCount); + } + + // Fills the cache from Position on, with at least byteCount bytes. + [MethodImpl(MethodImplOptions.NoInlining)] + private void FillBuffer(int byteCount) + { + EnsureAvailable(byteCount); + + _buffer ??= BufferPool.Rent(_bufferSize); + // emptied first: if the read throws, the cache holds nothing instead of stale data + _bufferLength = 0; + _bufferOffset = _position; + // not past Length, so that a file that grew after it was opened cannot serve reads beyond it from the cache + int fillLength = (int)Math.Min(_buffer.Length, LengthLong - _position); + _bufferLength = ReadAtLeast(_buffer.AsSpan(0, fillLength), _position, byteCount); + + if (_bufferLength < byteCount) + ThrowHelper.ThrowEndOfStream(); // the file got shorter after it was opened + } + + // Reads dest.Length bytes from Position on, past the cache. The callers have checked that they are available. + private void ReadDirect(Span dest) + { + if (ReadAtLeast(dest, _position, dest.Length) < dest.Length) + ThrowHelper.ThrowEndOfStream(); // the file got shorter after it was opened + } + + // RandomAccess.Read may return fewer bytes than requested, so this reads until minimumBytes or the end of the file. + private int ReadAtLeast(Span buffer, long fileOffset, int minimumBytes) + { + int totalRead = 0; + while (totalRead < minimumBytes) + { + int bytesRead = RandomAccess.Read(_handle, buffer[totalRead..], fileOffset + totalRead); + if (bytesRead == 0) + break; + totalRead += bytesRead; + } + + return totalRead; + } + + // Reads dest.Length bytes and moves Position past them. The callers have checked that they are available. + private void ReadBytesTo(Span dest) + { + if (dest.Length <= _bufferSize) + PeekBuffered(dest.Length).CopyTo(dest); + else + ReadDirect(dest); + + _position += dest.Length; } - public override T Read() where T : struct + public override T Read() { int size = Unsafe.SizeOf(); - EnsureBufferAllocated(size); - T result = _bufferReader.Read(); - PositionLong += size; + T result; + + if (size <= _bufferSize) + { + result = MemoryMarshal.Read(PeekBuffered(size)); + } + else + { + EnsureAvailable(size); + Unsafe.SkipInit(out result); + ReadDirect(MemoryMarshal.AsBytes(new Span(ref result))); + } + + _position += size; return result; } - public override void Read(Span dest) where T : struct + public override void Read(Span dest) { - if (dest.IsEmpty) - return; - - int size = Unsafe.SizeOf() * dest.Length; - EnsureBufferAllocated(size); - _bufferReader.Read(dest); - PositionLong += size; + // checked as a whole, so that a destination read in parts cannot fail half-way for lack of data + EnsureAvailable((long)dest.Length * Unsafe.SizeOf()); + ReadElements(dest); } - public override string ReadString(int length, Encoding enc) + // Fills dest and moves Position past it. The callers have checked that the bytes are available. + private void ReadElements(Span dest) where T : unmanaged { - return ReadString(length, enc, false); + // MemoryMarshal.AsBytes covers at most int.MaxValue bytes, so a larger destination is read in parts + int chunkLength = int.MaxValue / Unsafe.SizeOf(); + while (dest.Length > chunkLength) + { + ReadBytesTo(MemoryMarshal.AsBytes(dest[..chunkLength])); + dest = dest[chunkLength..]; + } + + ReadBytesTo(MemoryMarshal.AsBytes(dest)); } - private string ReadString(int length, Encoding enc, bool trimNull) + public override string ReadString(int length, Encoding enc) { - EnsureBufferAllocated(length); - string result = _bufferReader.ReadString(length, enc, trimNull); - PositionLong += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentNullException.ThrowIfNull(enc); + string result; + + if (length <= _bufferSize) + { + result = enc.GetString(PeekBuffered(length)); + } + else + { + EnsureAvailable(length); // before renting + using TempBuffer buffer = new(length, default); + ReadDirect(buffer.Span); + result = enc.GetString(buffer.Span); + } + + _position += length; return result; } public override string ReadFString() { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization int length = Read(); if (length == 0) return string.Empty; - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); + // the terminator is read as well, so that a missing one throws + int byteCount = ReaderHelpers.GetFStringByteCount(length); + string result; - int pLength = length * -sizeof(char); - string result = ReadString(pLength - sizeof(char), Encoding.Unicode, false); - PositionLong += sizeof(char); - return result; + if (byteCount <= _bufferSize) + { + result = ReaderHelpers.DecodeFString(PeekBuffered(byteCount), length); } else { - string result = ReadString(length, Encoding.UTF8, true); - return result; + EnsureAvailable(byteCount); // before renting + using TempBuffer buffer = new(byteCount, default); + ReadDirect(buffer.Span); + result = ReaderHelpers.DecodeFString(buffer.Span, length); } + + _position += byteCount; + return result; } - public override T[] ReadArray(int length) where T : struct + public override T[] ReadArray(int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that the byte count cannot wrap around, and checked before allocating + EnsureAvailable((long)length * Unsafe.SizeOf()); if (length == 0) return []; + ReaderHelpers.ValidateArrayLength(length); - int size = length * Unsafe.SizeOf(); - EnsureBufferAllocated(size); - T[] result = _bufferReader.ReadArray(length); - - PositionLong += size; + T[] result = GC.AllocateUninitializedArray(length); + ReadElements(result); return result; } + // With useSharedArrayPool, the array comes from ArrayPool.Shared and may be longer than length; only the + // first length bytes hold data. public byte[] ReadBytes(int length, bool useSharedArrayPool = false) { - byte[] buffer = useSharedArrayPool - ? ArrayPool.Shared.Rent(length) - : GC.AllocateUninitializedArray(length); - EnsureBufferAllocated(length); - _bufferReader.Read(buffer.AsSpan(0, length)); - PositionLong += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + EnsureAvailable(length); + + if (!useSharedArrayPool) + { + byte[] result = GC.AllocateUninitializedArray(length); + ReadBytesTo(result); + return result; + } + + byte[] buffer = ArrayPool.Shared.Rent(length); + try + { + ReadBytesTo(buffer.AsSpan(0, length)); + } + catch + { + ArrayPool.Shared.Return(buffer); + throw; + } + return buffer; } - public byte[] ReadBytes(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current, bool useSharedArrayPool = false) + public byte[] ReadBytes(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current, bool useSharedArrayPool = false) where TOffset : IBinaryInteger { - Seek(offset, origin); + SeekVoid(offset, origin); return ReadBytes(length, useSharedArrayPool); } - protected virtual void Dispose(bool disposing) + protected override void Dispose(bool disposing) { if (disposing) { - if (_bufferPosition != -1) - ArrayPool.Shared.Return(_buffer); - _handle.Dispose(); + // The cache is emptied before its array goes back to the pool, so that no read can see it afterwards. + // An offset of -1 makes even zero-byte reads miss it, which then throw ObjectDisposedException. + _bufferLength = 0; + _bufferOffset = -1; + if (_buffer is not null) + { + BufferPool.Return(_buffer); + _buffer = null; + } + + if (!_leaveOpen) + _handle.Dispose(); } } - - public override void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } } diff --git a/src/GenericReader/GenericReaderBase.cs b/src/GenericReader/GenericReaderBase.cs index e288f0e..ab5fd95 100644 --- a/src/GenericReader/GenericReaderBase.cs +++ b/src/GenericReader/GenericReaderBase.cs @@ -1,11 +1,26 @@ -using System.Diagnostics; -using System.Numerics; +using System.Numerics; using System.Text; namespace GenericReader; +// After Dispose, every read and every access to the position (Position, Seek, SetPosition) throws +// ObjectDisposedException; Length stays available. public abstract class GenericReaderBase : IGenericReader { + private bool _disposed; + + private protected void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); + + public long MaxArrayAllocationSize + { + get; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + field = value; + } + } = IGenericReader.DefaultMaxArrayAllocationSize; + public abstract int Position { get; set; } public abstract long PositionLong { get; set; } public abstract TPosition GetPosition() where TPosition : IBinaryInteger; @@ -17,38 +32,50 @@ public abstract class GenericReaderBase : IGenericReader public abstract long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; public abstract void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; - public abstract T Read() where T : struct; + // A length prefix in the data: a negative one means the data is corrupted. + private protected int ReadLength() + { + int length = Read(); + if (length < 0) + ThrowHelper.ThrowNegativeLength(length); + return length; + } + + // Reading at a position past the end fails, even when no bytes are read. + private protected void EnsureNotPastEnd() + { + if (PositionLong > LengthLong) + ThrowHelper.ThrowEndOfStream(); + } + + public abstract T Read() where T : unmanaged; public T Read(TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct where TOffset : IBinaryInteger + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); return Read(); } - public abstract void Read(Span dest) where T : struct; + public abstract void Read(Span dest) where T : unmanaged; public void Read(Span dest, TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct where TOffset : IBinaryInteger + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); Read(dest); } - public bool ReadBoolean() - { - uint num = Read(); - Debug.Assert(num is 0 or 1); - return num != 0; - } + public bool ReadBoolean() => Read() != 0; public bool ReadBoolean(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadBoolean(); } + public string ReadString(Encoding enc) { - int length = Read(); - return ReadString(length, enc); + ArgumentNullException.ThrowIfNull(enc); + return ReadString(ReadLength(), enc); } public string ReadString(Encoding enc, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger @@ -56,28 +83,24 @@ public string ReadString(Encoding enc, TOffset offset, SeekOrigin origi SeekVoid(offset, origin); return ReadString(enc); } - public abstract string ReadString(int length, Encoding enc); + public abstract string ReadString(int length, Encoding enc); public string ReadString(int length, Encoding enc, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadString(length, enc); } - public abstract string ReadFString(); + public abstract string ReadFString(); public string ReadFString(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadFString(); } - public string[] ReadFStringArray() - { - int length = Read(); - return ReadFStringArray(length); - } + public string[] ReadFStringArray() => ReadFStringArray(ReadLength()); public string[] ReadFStringArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { @@ -87,10 +110,15 @@ public string[] ReadFStringArray(TOffset offset, SeekOrigin origin = Se public string[] ReadFStringArray(int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // every FString takes at least 4 bytes (its length), so a larger count is rejected before allocating + if ((long)length * sizeof(int) > LengthLong - PositionLong) + ThrowHelper.ThrowEndOfStream(); if (length == 0) return []; + ReaderHelpers.ValidateArrayLength(length); - string[] result = GC.AllocateUninitializedArray(length); + string[] result = new string[length]; for (int i = 0; i < length; i++) result[i] = ReadFString(); @@ -105,22 +133,18 @@ public string[] ReadFStringArray(int length, TOffset offset, SeekOrigin return ReadFStringArray(length); } - public T[] ReadArray() where T : struct - { - int length = Read(); - return ReadArray(length); - } + public T[] ReadArray() where T : unmanaged => ReadArray(ReadLength()); public T[] ReadArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadArray(); } - public abstract T[] ReadArray(int length) where T : struct; + public abstract T[] ReadArray(int length) where T : unmanaged; public T[] ReadArray(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); @@ -129,37 +153,37 @@ public T[] ReadArray(int length, TOffset offset, SeekOrigin origin = public T[] ReadArray(Func getter) { - int length = Read(); - return ReadArray(length, getter); + ArgumentNullException.ThrowIfNull(getter); + return ReadArray(ReadLength(), getter); } - public T[] ReadArray(Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - int length = Read(); - return ReadArray(length, getter); + return ReadArray(getter); } public T[] ReadArray(Func getter) { - int length = Read(); - return ReadArray(length, getter); + ArgumentNullException.ThrowIfNull(getter); + return ReadArray(ReadLength(), getter); } - - public T[] ReadArray(Func getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) + public T[] ReadArray(Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - int length = Read(); - return ReadArray(length, getter); + return ReadArray(getter); } public T[] ReadArray(int length, Func getter) { + ArgumentNullException.ThrowIfNull(getter); + ReaderHelpers.ValidateGetterArrayLength(length, MaxArrayAllocationSize); if (length == 0) + { + EnsureNotPastEnd(); return []; + } T[] result = GC.AllocateUninitializedArray(length); @@ -178,8 +202,13 @@ public T[] ReadArray(int length, Func getter, TOffset offset, See public T[] ReadArray(int length, Func getter) { + ArgumentNullException.ThrowIfNull(getter); + ReaderHelpers.ValidateGetterArrayLength(length, MaxArrayAllocationSize); if (length == 0) + { + EnsureNotPastEnd(); return []; + } T[] result = GC.AllocateUninitializedArray(length); @@ -189,13 +218,23 @@ public T[] ReadArray(int length, Func getter) return result; } - public T[] ReadArray(int length, Func getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) + public T[] ReadArray(int length, Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadArray(length, getter); } - public abstract void Dispose(); + // Runs once, however often Dispose is called. + protected virtual void Dispose(bool disposing) { } + + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + + Dispose(true); + GC.SuppressFinalize(this); + } } diff --git a/src/GenericReader/GenericSpanReader.cs b/src/GenericReader/GenericSpanReader.cs index f554dd2..42f9c4a 100644 --- a/src/GenericReader/GenericSpanReader.cs +++ b/src/GenericReader/GenericSpanReader.cs @@ -10,93 +10,143 @@ namespace GenericReader; public delegate T GetterFunc(ref GenericSpanReader reader); private const string ObsoleteMessage = "This method cannot be used since this reader is a ref struct"; - private readonly Span _span; + private readonly ReadOnlySpan _span; + private int _position; + // stored as the difference to the default, so that default(GenericSpanReader) has the default limit too + private long _maxArrayAllocationSizeOffset; - public GenericSpanReader(byte[] buffer, int start, int length) : this(new Span(buffer, start, length)) { } - public GenericSpanReader(byte[] buffer) : this(new Span(buffer)) { } + public GenericSpanReader(byte[] buffer, int start, int length) : this(new ReadOnlySpan(buffer, start, length)) { } + public GenericSpanReader(byte[] buffer) : this(new ReadOnlySpan(buffer)) { } public GenericSpanReader(ReadOnlyMemory memory) : this(memory.Span) { } - public GenericSpanReader(Memory memory) : this(memory.Span) { } - public GenericSpanReader(ReadOnlySpan span) : this(MemoryMarshal.CreateSpan(ref MemoryMarshal.GetReference(span), span.Length)) { } - public GenericSpanReader(Span span) + public GenericSpanReader(ReadOnlySpan span) { _span = span; - LengthLong = Length = span.Length; } - public int Position { get; set; } + public long MaxArrayAllocationSize + { + readonly get => IGenericReader.DefaultMaxArrayAllocationSize + _maxArrayAllocationSizeOffset; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + _maxArrayAllocationSizeOffset = value - IGenericReader.DefaultMaxArrayAllocationSize; + } + } + + public int Position + { + readonly get => _position; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + _position = value; + } + } public long PositionLong { - get => Position; - set => Position = int.CreateChecked(value); + readonly get => _position; + set => _position = ReaderHelpers.ToInt32Position(value); } - public TPosition GetPosition() where TPosition : IBinaryInteger + public readonly TPosition GetPosition() where TPosition : IBinaryInteger { - return TPosition.CreateChecked(Position); + return TPosition.CreateChecked(_position); } public void SetPosition(TPosition position) where TPosition : IBinaryInteger { - Position = int.CreateChecked(position); + _position = (int)ReaderHelpers.ToPosition(position, int.MaxValue); } - public int Length { get; } - public long LengthLong { get; } + public readonly int Length => _span.Length; + public readonly long LengthLong => _span.Length; - public TLength GetLength() where TLength : IBinaryInteger + public readonly TLength GetLength() where TLength : IBinaryInteger { - return TLength.CreateChecked(Length); + return TLength.CreateChecked(_span.Length); } public int Seek(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - return Position; + return _position; } public long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - return Position; + return _position; } public void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { - Position = origin switch - { - SeekOrigin.Begin => int.CreateChecked(offset), - SeekOrigin.Current => Position + int.CreateChecked(offset), - SeekOrigin.End => Length + int.CreateChecked(offset), - _ => throw new ArgumentOutOfRangeException(nameof(origin), origin, null) - }; + _position = (int)ReaderHelpers.GetSeekTarget(offset, origin, _position, _span.Length, int.MaxValue); + } + + // Throws unless byteCount (not negative) bytes are left from Position. Position is never negative, so the + // subtraction cannot overflow; it is negative when Position lies past the end. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly void EnsureAvailable(long byteCount) + { + if (byteCount > _span.Length - _position) + ThrowHelper.ThrowEndOfStream(); + } + + // The next byteCount (not negative) bytes, without moving Position. The two Peek overloads are the only places + // that create spans over the data, so no span is created without a bounds check. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly ReadOnlySpan Peek(int byteCount) + { + // both are at most int.MaxValue, so the unsigned sum cannot wrap around + if ((uint)_position + (uint)byteCount > (uint)_span.Length) + ThrowHelper.ThrowEndOfStream(); + + return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.Add(ref MemoryMarshal.GetReference(_span), (nuint)(uint)_position), byteCount); + } + + // Peek for a computed byte count such as length * SizeOf, which may not fit in an int. + private readonly ReadOnlySpan Peek(long byteCount) + { + EnsureAvailable(byteCount); + // at most the length of the data now, so the cast cannot truncate + return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.Add(ref MemoryMarshal.GetReference(_span), (nuint)(uint)_position), (int)byteCount); + } + + // A length prefix in the data: a negative one means the data is corrupted. + private int ReadLength() + { + int length = Read(); + if (length < 0) + ThrowHelper.ThrowNegativeLength(length); + return length; } - public T Read() where T : struct + public T Read() where T : unmanaged { - T result = Unsafe.ReadUnaligned(ref _span[Position]); int size = Unsafe.SizeOf(); - Position += size; + T result = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(Peek(size))); + _position += size; return result; } public T Read(TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct where TOffset : IBinaryInteger + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); return Read(); } - public void Read(Span dest) where T : struct + public void Read(Span dest) where T : unmanaged { - int size = Unsafe.SizeOf(); - Span span = MemoryMarshal.CreateSpan(ref Unsafe.As(ref dest[0]), dest.Length * size); - _span.Slice(Position, span.Length).CopyTo(span); - Position += span.Length; + // 64-bit: a destination of more than int.MaxValue bytes cannot fit in the data, but must not overflow either + ReadOnlySpan source = Peek((long)dest.Length * Unsafe.SizeOf()); + source.CopyTo(MemoryMarshal.AsBytes(dest)); + _position += source.Length; } public void Read(Span dest, TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct where TOffset : IBinaryInteger + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); Read(dest); @@ -112,8 +162,8 @@ public bool ReadBoolean(TOffset offset, SeekOrigin origin = SeekOrigin. public string ReadString(Encoding enc) { - int length = Read(); - return ReadString(length, enc); + ArgumentNullException.ThrowIfNull(enc); + return ReadString(ReadLength(), enc); } public string ReadString(Encoding enc, TOffset offset, SeekOrigin origin = SeekOrigin.Current) @@ -132,38 +182,23 @@ public string ReadString(int length, Encoding enc, TOffset offset, Seek public string ReadString(int length, Encoding enc) { - string result = enc.GetString(_span.Slice(Position, length)); - Position += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentNullException.ThrowIfNull(enc); + string result = enc.GetString(Peek(length)); + _position += length; return result; } public string ReadFString() { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization int length = Read(); if (length == 0) return string.Empty; - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); - - int pLength = length * -sizeof(char); - Span span = _span.Slice(Position, pLength - sizeof(char)); - string result = Encoding.Unicode.GetString(span); - Position += pLength; - return result; - } - else - { - Span span = _span.Slice(Position, length).TrimEnd(byte.MinValue); - string result = Encoding.UTF8.GetString(span); - Position += length; - return result; - } + int byteCount = ReaderHelpers.GetFStringByteCount(length); + string result = ReaderHelpers.DecodeFString(Peek(byteCount), length); + _position += byteCount; + return result; } public string ReadFString(TOffset offset, SeekOrigin origin = SeekOrigin.Current) @@ -173,11 +208,7 @@ public string ReadFString(TOffset offset, SeekOrigin origin = SeekOrigi return ReadFString(); } - public string[] ReadFStringArray() - { - int length = Read(); - return ReadFStringArray(length); - } + public string[] ReadFStringArray() => ReadFStringArray(ReadLength()); public string[] ReadFStringArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger @@ -188,10 +219,13 @@ public string[] ReadFStringArray(TOffset offset, SeekOrigin origin = Se public string[] ReadFStringArray(int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // every FString takes at least 4 bytes (its length), so a larger count is rejected before allocating + EnsureAvailable((long)length * sizeof(int)); if (length == 0) return []; - string[] result = GC.AllocateUninitializedArray(length); + string[] result = new string[length]; for (int i = 0; i < length; i++) result[i] = ReadFString(); @@ -206,77 +240,32 @@ public string[] ReadFStringArray(int length, TOffset offset, SeekOrigin return ReadFStringArray(length); } - /*public FStringMemory ReadFStringMemory() - { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization - var length = Read(); - if (length == 0) - return default; - - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); - - var pLength = length * -sizeof(char); - var memory = _memory.Slice(Position, pLength - sizeof(char)); - Position += pLength; - return new FStringMemory(memory, true); - } - else - { - var memory = _memory.Slice(Position, length - 1); - Position += length; - return new FStringMemory(memory, false); - } - } - - public FStringMemory[] ReadFStringMemoryArray(int length) - { - var result = new FStringMemory[length]; - - for (var i = 0; i < length; i++) - result[i] = ReadFStringMemory(); - - return result; - } - - public FStringMemory[] ReadFStringMemoryArray() - { - var length = Read(); - return ReadFStringMemoryArray(length); - }*/ - - public T[] ReadArray() where T : struct - { - int length = Read(); - return ReadArray(length); - } + public T[] ReadArray() where T : unmanaged => ReadArray(ReadLength()); public T[] ReadArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadArray(); } - public T[] ReadArray(int length) where T : struct + public T[] ReadArray(int length) where T : unmanaged { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that the byte count cannot wrap around, and checked before allocating + ReadOnlySpan source = Peek((long)length * Unsafe.SizeOf()); if (length == 0) return []; - int size = length * Unsafe.SizeOf(); T[] result = GC.AllocateUninitializedArray(length); - Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref result[0]), ref _span[Position], (uint)size); - Position += size; + source.CopyTo(MemoryMarshal.AsBytes(result.AsSpan())); + _position += source.Length; return result; } public T[] ReadArray(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current) - where T : struct + where T : unmanaged where TOffset : IBinaryInteger { SeekVoid(offset, origin); @@ -285,54 +274,39 @@ public T[] ReadArray(int length, TOffset offset, SeekOrigin origin = public T[] ReadArray(Func getter) { - int length = Read(); - return ReadArray(length, getter); + ArgumentNullException.ThrowIfNull(getter); + return ReadArray(ReadLength(), getter); } public T[] ReadArray(Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - int length = Read(); - return ReadArray(length, getter); - } - - [Obsolete(ObsoleteMessage, true)] - public T[] ReadArray(Func getter) - { - int length = Read(); - return ReadArray(length, getter); + return ReadArray(getter); } public T[] ReadArray(GetterFunc getter) { - int length = Read(); - return ReadArray(length, getter); - } - - [Obsolete(ObsoleteMessage, true)] - public T[] ReadArray(Func getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) - where TOffset : IBinaryInteger - { - SeekVoid(offset, origin); - int length = Read(); - return ReadArray(length, getter); + ArgumentNullException.ThrowIfNull(getter); + return ReadArray(ReadLength(), getter); } - public T[] ReadArray(GetterFunc getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) + public T[] ReadArray(GetterFunc getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); - int length = Read(); - return ReadArray(length, getter); + return ReadArray(getter); } public T[] ReadArray(int length, Func getter) { + ArgumentNullException.ThrowIfNull(getter); + ReaderHelpers.ValidateGetterArrayLength(length, MaxArrayAllocationSize); if (length == 0) + { + EnsureAvailable(0); return []; + } T[] result = GC.AllocateUninitializedArray(length); @@ -349,24 +323,15 @@ public T[] ReadArray(int length, Func getter, TOffset offset, See return ReadArray(length, getter); } - [Obsolete(ObsoleteMessage, true)] - public T[] ReadArray(int length, Func getter) - { - if (length == 0) - return []; - - T[] result = GC.AllocateUninitializedArray(length); - - /*for (var i = 0; i < length; i++) - result[i] = getter(this);*/ - - return result; - } - public T[] ReadArray(int length, GetterFunc getter) { + ArgumentNullException.ThrowIfNull(getter); + ReaderHelpers.ValidateGetterArrayLength(length, MaxArrayAllocationSize); if (length == 0) + { + EnsureAvailable(0); return []; + } T[] result = GC.AllocateUninitializedArray(length); @@ -375,69 +340,67 @@ public T[] ReadArray(int length, GetterFunc getter) return result; } - - [Obsolete(ObsoleteMessage, true)] - public T[] ReadArray(int length, Func getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) - where TOffset : IBinaryInteger - { - SeekVoid(offset, origin); - return ReadArray(length, getter); - } - public T[] ReadArray(int length, GetterFunc getter, TOffset offset, - SeekOrigin origin = SeekOrigin.Current) + public T[] ReadArray(int length, GetterFunc getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger { SeekVoid(offset, origin); return ReadArray(length, getter); } - public GenericSpanReader Slice(int start, bool sliceAtPosition = false) - { - int sliceStart = sliceAtPosition ? start + Position : start; - return new GenericSpanReader(_span.Slice(sliceStart)); - } + // A ref struct cannot be passed as IGenericReader, so these getters cannot be called. They throw instead of being + // left out because the interface requires them; [Obsolete] does not stop calls through a generic TReader. + [Obsolete(ObsoleteMessage, true)] + public readonly T[] ReadArray(Func getter) => throw new NotSupportedException(ObsoleteMessage); + + [Obsolete(ObsoleteMessage, true)] + public readonly T[] ReadArray(Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) + where TOffset : IBinaryInteger => throw new NotSupportedException(ObsoleteMessage); + + [Obsolete(ObsoleteMessage, true)] + public readonly T[] ReadArray(int length, Func getter) => throw new NotSupportedException(ObsoleteMessage); - public GenericSpanReader Slice(int start, int length, bool sliceAtPosition = false) + [Obsolete(ObsoleteMessage, true)] + public readonly T[] ReadArray(int length, Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) + where TOffset : IBinaryInteger => throw new NotSupportedException(ObsoleteMessage); + + public readonly GenericSpanReader Slice(int start, bool sliceAtPosition = false) { - int sliceStart = sliceAtPosition ? start + Position : start; - return new GenericSpanReader(_span.Slice(sliceStart, length)); + int sliceStart = sliceAtPosition ? start + _position : start; + return new GenericSpanReader(_span.Slice(sliceStart)) { _maxArrayAllocationSizeOffset = _maxArrayAllocationSizeOffset }; } - public Span AsSpan(bool sliceAtPosition = false) + public readonly GenericSpanReader Slice(int start, int length, bool sliceAtPosition = false) { - return sliceAtPosition ? _span.Slice(Position) : _span; + int sliceStart = sliceAtPosition ? start + _position : start; + return new GenericSpanReader(_span.Slice(sliceStart, length)) { _maxArrayAllocationSizeOffset = _maxArrayAllocationSizeOffset }; } - public Span ReadSpan() + public readonly ReadOnlySpan AsSpan(bool sliceAtPosition = false) { - int length = Read(); - return ReadSpan(length); + return sliceAtPosition ? _span.Slice(_position) : _span; } - public Span ReadSpan(int length) + public ReadOnlySpan ReadSpan() => ReadSpan(ReadLength()); + + public ReadOnlySpan ReadSpan(int length) { - Span result = _span.Slice(Position, length); - Position += length; + ArgumentOutOfRangeException.ThrowIfNegative(length); + ReadOnlySpan result = Peek(length); + _position += length; return result; } - public Span ReadSpan() where T : struct - { - int length = Read(); - return ReadSpan(length); - } + public ReadOnlySpan ReadSpan() where T : unmanaged => ReadSpan(ReadLength()); - public Span ReadSpan(int length) where T : struct + public ReadOnlySpan ReadSpan(int length) where T : unmanaged { - int size = length * Unsafe.SizeOf(); - Span span = _span.Slice(Position, size); - ref T reference = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); - Span resultSpan = MemoryMarshal.CreateSpan(ref reference, length); - Position += size; - return resultSpan; + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that a wrapped byte count cannot pass the check while the span keeps the full length + ReadOnlySpan source = Peek((long)length * Unsafe.SizeOf()); + _position += source.Length; + return MemoryMarshal.Cast(source); } - public void Dispose() { } + public readonly void Dispose() { } } diff --git a/src/GenericReader/GenericStreamReader.cs b/src/GenericReader/GenericStreamReader.cs index eaf3f5f..d4652d9 100644 --- a/src/GenericReader/GenericStreamReader.cs +++ b/src/GenericReader/GenericStreamReader.cs @@ -6,29 +6,55 @@ namespace GenericReader; +// Reads from a seekable stream. Every read is checked against Length before anything is allocated or read, and +// a read that fails puts the stream back where it started, even when the stream ends before the Length it reported +// (the destination may then hold part of the data). public class GenericStreamReader : GenericReaderBase { private readonly Stream _stream; + private readonly bool _leaveOpen; public GenericStreamReader(string filePath) : this(File.OpenRead(filePath)) { } - public GenericStreamReader(Stream stream) + + // Unless leaveOpen, the reader owns the stream: it disposes it, also when this constructor fails after the + // argument was validated (for example for a stream that cannot seek). + public GenericStreamReader(Stream stream, bool leaveOpen = false) { + ArgumentNullException.ThrowIfNull(stream); _stream = stream; - long streamLength = _stream.Length; - LengthLong = streamLength; - Length = unchecked((int)streamLength); + _leaveOpen = leaveOpen; + + try + { + LengthLong = stream.Length; + } + catch + { + if (!leaveOpen) + stream.Dispose(); + throw; + } } public override int Position { - get => unchecked((int)PositionLong); + get => int.CreateChecked(PositionLong); set => PositionLong = value; } public override long PositionLong { - get => _stream.Position; - set => _stream.Position = value; + get + { + ThrowIfDisposed(); + return _stream.Position; + } + set + { + ThrowIfDisposed(); + ArgumentOutOfRangeException.ThrowIfNegative(value); + _stream.Position = value; + } } public override TPosition GetPosition() @@ -38,10 +64,11 @@ public override TPosition GetPosition() public override void SetPosition(TPosition position) { - PositionLong = long.CreateChecked(position); + ThrowIfDisposed(); + _stream.Position = ReaderHelpers.ToPosition(position, long.MaxValue); } - public override int Length { get; } + public override int Length => int.CreateChecked(LengthLong); public override long LengthLong { get; } public override TLength GetLength() @@ -51,157 +78,171 @@ public override TLength GetLength() public override int Seek(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - return unchecked((int)SeekLong(offset, origin)); + // checked against int.MaxValue before the stream moves + long target = ReaderHelpers.GetSeekTarget(offset, origin, PositionLong, LengthLong, int.MaxValue); + _stream.Position = target; + return (int)target; } public override long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - return _stream.Seek(long.CreateChecked(offset), origin); + SeekVoid(offset, origin); + return PositionLong; } public override void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) { - _stream.Seek(long.CreateChecked(offset), origin); + _stream.Position = ReaderHelpers.GetSeekTarget(offset, origin, PositionLong, LengthLong, long.MaxValue); } - public override T Read() where T : struct + // Throws unless byteCount (not negative) bytes are left from Position, which it returns. + private long EnsureAvailable(long byteCount) { - int size = Unsafe.SizeOf(); - T result; + long position = PositionLong; + if (byteCount > LengthLong - position) + ThrowHelper.ThrowEndOfStream(); + return position; + } - if (size > Constants.MaxStackSize) + // Fills dest, or throws with the stream back at start, where dest begins. The callers have checked that the bytes + // are available and pass the position that EnsureAvailable returned, so that it is not queried again. + private void ReadExactly(Span dest, long start) + { + try { - byte[] buffer = ArrayPool.Shared.Rent(size); - _stream.ReadExactly(buffer, 0, size); - result = Unsafe.ReadUnaligned(ref buffer[0]); - ArrayPool.Shared.Return(buffer); + _stream.ReadExactly(dest); } - else + catch { - Span span = stackalloc byte[size]; - _stream.ReadExactly(span); - result = Unsafe.ReadUnaligned(ref span[0]); + _stream.Position = start; // the stream ended before the Length it reported + throw; } - - return result; } - public override void Read(Span dest) where T : struct + // ReadExactly for elements, which may take more than int.MaxValue bytes. + private void ReadExactly(Span dest, long start) where T : unmanaged { - if (dest.IsEmpty) - return; - - int size = Unsafe.SizeOf(); - Span span = MemoryMarshal.CreateSpan(ref Unsafe.As(ref dest[0]), dest.Length * size); - _stream.ReadExactly(span); + try + { + // MemoryMarshal.AsBytes covers at most int.MaxValue bytes, so a larger destination is read in parts + int chunkLength = int.MaxValue / Unsafe.SizeOf(); + while (dest.Length > chunkLength) + { + _stream.ReadExactly(MemoryMarshal.AsBytes(dest[..chunkLength])); + dest = dest[chunkLength..]; + } + + _stream.ReadExactly(MemoryMarshal.AsBytes(dest)); + } + catch + { + _stream.Position = start; // the stream ended before the Length it reported + throw; + } } - public override string ReadString(int length, Encoding enc) + public override T Read() { - return ReadString(length, enc, false); + long start = EnsureAvailable(Unsafe.SizeOf()); + Unsafe.SkipInit(out T result); + ReadExactly(MemoryMarshal.AsBytes(new Span(ref result)), start); + return result; } - private string ReadString(int length, Encoding enc, bool trimNull) + public override void Read(Span dest) { - string result; - - if (length > Constants.MaxStackSize) - { - byte[] buffer = ArrayPool.Shared.Rent(length); - var span = new Span(buffer, 0, length); - _stream.ReadExactly(span); - if (trimNull) - span = span.TrimEnd(byte.MinValue); - result = enc.GetString(span); - ArrayPool.Shared.Return(buffer); - } - else - { - Span span = stackalloc byte[length]; - _stream.ReadExactly(span); - if (trimNull) - span = span.TrimEnd(byte.MinValue); - result = enc.GetString(span); - } + ReadExactly(dest, EnsureAvailable((long)dest.Length * Unsafe.SizeOf())); + } - return result; + public override string ReadString(int length, Encoding enc) + { + ArgumentOutOfRangeException.ThrowIfNegative(length); + ArgumentNullException.ThrowIfNull(enc); + return ReadAndDecode(length, enc, fStringLength: 0); } public override string ReadFString() { - // > 0 for ANSICHAR, < 0 for UCS2CHAR serialization int length = Read(); if (length == 0) return string.Empty; - // 1 byte/char is removed because of null terminator ('\0') - if (length < 0) // LoadUCS2Char, Unicode, 16-bit, fixed-width - { - // If length cannot be negated due to integer overflow, Ar is corrupted. - if (length == int.MinValue) - throw new ArgumentOutOfRangeException(nameof(length), "Archive is corrupted"); + // the terminator is read as well, so that a missing one throws + return ReadAndDecode(ReaderHelpers.GetFStringByteCount(length), Encoding.UTF8, length); + } - int pLength = length * -sizeof(char); - string result = ReadString(pLength - sizeof(char), Encoding.Unicode, false); - PositionLong += sizeof(char); - return result; + // Reads byteCount bytes and decodes them with enc, or as the FString of fStringLength when that is not 0. Throws + // with the stream back where it started, also when decoding fails, like the other readers. + private string ReadAndDecode(int byteCount, Encoding enc, int fStringLength) + { + long start = EnsureAvailable(byteCount); + using TempBuffer buffer = new(byteCount, byteCount <= Constants.MaxStackSize ? stackalloc byte[byteCount] : default); + try + { + _stream.ReadExactly(buffer.Span); + return fStringLength == 0 + ? enc.GetString(buffer.Span) + : ReaderHelpers.DecodeFString(buffer.Span, fStringLength); } - else + catch { - string result = ReadString(length, Encoding.UTF8, true); - return result; + _stream.Position = start; + throw; } } - public override T[] ReadArray(int length) where T : struct + public override T[] ReadArray(int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // 64-bit, so that the byte count cannot wrap around, and checked before allocating + long start = EnsureAvailable((long)length * Unsafe.SizeOf()); if (length == 0) return []; + ReaderHelpers.ValidateArrayLength(length); - int size = length * Unsafe.SizeOf(); T[] result = GC.AllocateUninitializedArray(length); + ReadExactly(result, start); + return result; + } + + // With useSharedArrayPool, the array comes from ArrayPool.Shared and may be longer than length; only the + // first length bytes hold data. + public byte[] ReadBytes(int length, bool useSharedArrayPool = false) + { + ArgumentOutOfRangeException.ThrowIfNegative(length); + long start = EnsureAvailable(length); - if (size > Constants.MaxStackSize) + if (!useSharedArrayPool) { - byte[] buffer = ArrayPool.Shared.Rent(size); - _stream.ReadExactly(buffer, 0, size); - Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref result[0]), ref buffer[0], (uint)size); - ArrayPool.Shared.Return(buffer); + byte[] result = GC.AllocateUninitializedArray(length); + ReadExactly(result, start); + return result; } - else + + byte[] buffer = ArrayPool.Shared.Rent(length); + try { - Span span = stackalloc byte[size]; - _stream.ReadExactly(span); - Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref result[0]), ref span[0], (uint)size); + ReadExactly(buffer.AsSpan(0, length), start); + } + catch + { + ArrayPool.Shared.Return(buffer); + throw; } - return result; - } - - public byte[] ReadBytes(int length, bool useSharedArrayPool = false) - { - byte[] buffer = useSharedArrayPool - ? ArrayPool.Shared.Rent(length) - : GC.AllocateUninitializedArray(length); - _stream.ReadExactly(buffer, 0, length); return buffer; } - public byte[] ReadBytes(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current, bool useSharedArrayPool = false) + public byte[] ReadBytes(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current, bool useSharedArrayPool = false) where TOffset : IBinaryInteger { - Seek(offset, origin); + SeekVoid(offset, origin); return ReadBytes(length, useSharedArrayPool); } - protected virtual void Dispose(bool disposing) - { - if (disposing) _stream?.Dispose(); - } - - public override void Dispose() + protected override void Dispose(bool disposing) { - Dispose(true); - GC.SuppressFinalize(this); + if (disposing && !_leaveOpen) + _stream.Dispose(); } } diff --git a/src/GenericReader/IGenericReader.cs b/src/GenericReader/IGenericReader.cs index 19f3b0f..fa4df50 100644 --- a/src/GenericReader/IGenericReader.cs +++ b/src/GenericReader/IGenericReader.cs @@ -16,9 +16,16 @@ public interface IGenericReader : IDisposable public static GenericSpanReader Create(Span span) => new(span); public static GenericFileReader Create(string filePath) => new(filePath); - public static GenericFileReader Create(SafeFileHandle handle) => new(handle); + public static GenericFileReader Create(SafeFileHandle handle, bool leaveOpen = false) => new(handle, leaveOpen: leaveOpen); - public static GenericStreamReader Create(Stream stream) => new(stream); + public static GenericStreamReader Create(Stream stream, bool leaveOpen = false) => new(stream, leaveOpen); + + // The array size in bytes that ReadArray with a getter allocates at most (256 MiB). Such arrays are allocated before + // any element is read, and an element may take no bytes at all, so their length cannot be checked against the data + // like the others. + public const long DefaultMaxArrayAllocationSize = 1L << 28; + + long MaxArrayAllocationSize { get; set; } int Position { get; set; } long PositionLong { get; set; } @@ -29,10 +36,11 @@ public interface IGenericReader : IDisposable TLength GetLength() where TLength : IBinaryInteger; int Seek(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; long SeekLong(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; - T Read() where T : struct; - T Read(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : struct where TOffset : IBinaryInteger; - void Read(Span dest) where T : struct; - void Read(Span dest, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : struct where TOffset : IBinaryInteger; + void SeekVoid(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; + T Read() where T : unmanaged; + T Read(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : unmanaged where TOffset : IBinaryInteger; + void Read(Span dest) where T : unmanaged; + void Read(Span dest, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : unmanaged where TOffset : IBinaryInteger; bool ReadBoolean(); bool ReadBoolean(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; string ReadString(Encoding enc); @@ -45,10 +53,10 @@ public interface IGenericReader : IDisposable string[] ReadFStringArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; string[] ReadFStringArray(int length); string[] ReadFStringArray(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; - T[] ReadArray() where T : struct; - T[] ReadArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : struct where TOffset : IBinaryInteger; - T[] ReadArray(int length) where T : struct; - T[] ReadArray(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : struct where TOffset : IBinaryInteger; + T[] ReadArray() where T : unmanaged; + T[] ReadArray(TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : unmanaged where TOffset : IBinaryInteger; + T[] ReadArray(int length) where T : unmanaged; + T[] ReadArray(int length, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where T : unmanaged where TOffset : IBinaryInteger; T[] ReadArray(Func getter); T[] ReadArray(Func getter, TOffset offset, SeekOrigin origin = SeekOrigin.Current) where TOffset : IBinaryInteger; T[] ReadArray(Func getter); diff --git a/src/GenericReader/ReaderHelpers.cs b/src/GenericReader/ReaderHelpers.cs new file mode 100644 index 0000000..cccf104 --- /dev/null +++ b/src/GenericReader/ReaderHelpers.cs @@ -0,0 +1,96 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Text; + +namespace GenericReader; + +internal static class ReaderHelpers +{ + // The readers over memory address at most int.MaxValue bytes. + public static int ToInt32Position(long value) + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, int.MaxValue); + return (int)value; + } + + // A position of any integer type, which has to lie in [0, maxPosition]. + public static long ToPosition(TPosition position, long maxPosition) + where TPosition : IBinaryInteger + { + // saturated to 128-bit, where every out-of-range value stays out of range + Int128 value = Int128.CreateSaturating(position); + if (value < 0 || value > maxPosition) + ThrowHelper.ThrowPositionOutOfRange(value, maxPosition); + + return (long)value; + } + + // The position that Seek moves to, which has to lie in [0, maxPosition]. + public static long GetSeekTarget(TOffset offset, SeekOrigin origin, long position, long length, long maxPosition) + where TOffset : IBinaryInteger + { + long start = origin switch + { + SeekOrigin.Begin => 0, + SeekOrigin.Current => position, + SeekOrigin.End => length, + _ => throw new ArgumentOutOfRangeException(nameof(origin), origin, null) + }; + + // 128-bit, so that neither a huge offset nor the addition can wrap around into the valid range + Int128 target = start + Int128.CreateSaturating(offset); + if (target < 0 || target > maxPosition) + ThrowHelper.ThrowSeekOutOfRange(target, maxPosition); + + return (long)target; + } + + // A getter array is allocated before any element is read, so its size is limited by maxByteCount instead of the data. + public static void ValidateGetterArrayLength(int length, long maxByteCount) + { + ArgumentOutOfRangeException.ThrowIfNegative(length); + // the size of a reference for reference types + long byteCount = (long)length * Unsafe.SizeOf(); + if (byteCount > maxByteCount) + ThrowHelper.ThrowArrayTooLarge(length, byteCount, maxByteCount); + ValidateArrayLength(length); + } + + // Even when the data holds enough bytes, no array can have more elements than this. + public static void ValidateArrayLength(int length) + { + if (length > Array.MaxLength) + ThrowHelper.ThrowArrayTooLong(length); + } + + // The buffer for the remaining bytes of a file or stream, which has to fit in an array. + public static byte[] AllocateBuffer(long length) + { + if (length > Array.MaxLength) + throw new IOException($"The data is too large ({length} bytes) to be loaded into a buffer."); + + // every byte gets read into it + return GC.AllocateUninitializedArray((int)Math.Max(length, 0)); + } + + // An Unreal FString is an int32 length followed by the characters and a null terminator. A positive length counts + // UTF-8 bytes, a negative one UTF-16 chars, both including the terminator. Returns the byte count of the characters + // and the terminator; length must not be 0. + public static int GetFStringByteCount(int length) + { + if (length > 0) + return length; + + // more chars than this do not fit in an int as bytes, and int.MinValue cannot be negated + if (length < -(int.MaxValue / sizeof(char))) + ThrowHelper.ThrowInvalidFStringLength(length); + + return -length * sizeof(char); + } + + // bytes holds GetFStringByteCount(length) bytes: the characters and the terminator + public static string DecodeFString(ReadOnlySpan bytes, int length) => length < 0 + ? Encoding.Unicode.GetString(bytes[..^sizeof(char)]) + : Encoding.UTF8.GetString(bytes.TrimEnd(byte.MinValue)); +} diff --git a/src/GenericReader/TempBuffer.cs b/src/GenericReader/TempBuffer.cs new file mode 100644 index 0000000..9cc67e7 --- /dev/null +++ b/src/GenericReader/TempBuffer.cs @@ -0,0 +1,34 @@ +using System.Buffers; + +namespace GenericReader; + +// length bytes of scratch memory: the given stack buffer when it is large enough, else an array rented from the pool +// that Dispose returns. +internal ref struct TempBuffer +{ + private byte[]? _rented; + + public Span Span { get; } + + public TempBuffer(int length, Span stackBuffer) + { + if (length <= stackBuffer.Length) + { + Span = stackBuffer[..length]; + } + else + { + _rented = ArrayPool.Shared.Rent(length); + Span = _rented.AsSpan(0, length); + } + } + + public void Dispose() + { + if (_rented is null) + return; + + ArrayPool.Shared.Return(_rented); + _rented = null; + } +} diff --git a/src/GenericReader/ThrowHelper.cs b/src/GenericReader/ThrowHelper.cs new file mode 100644 index 0000000..b7e44c4 --- /dev/null +++ b/src/GenericReader/ThrowHelper.cs @@ -0,0 +1,37 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace GenericReader; + +// Throws live outside the read methods so that those stay small enough to be inlined. +[StackTraceHidden] +internal static class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowEndOfStream() => throw new EndOfStreamException(); + + [DoesNotReturn] + public static void ThrowNegativeLength(int length) => + throw new InvalidDataException($"The data contains a negative length ({length})."); + + [DoesNotReturn] + public static void ThrowInvalidFStringLength(int length) => + throw new InvalidDataException($"The FString length {length} is out of range."); + + [DoesNotReturn] + public static void ThrowArrayTooLarge(int length, long byteCount, long maxByteCount) => + throw new InvalidDataException( + $"An array of {length} elements ({byteCount} bytes) exceeds MaxArrayAllocationSize ({maxByteCount} bytes)."); + + [DoesNotReturn] + public static void ThrowArrayTooLong(int length) => + throw new InvalidDataException($"The array length {length} exceeds Array.MaxLength ({Array.MaxLength})."); + + [DoesNotReturn] + public static void ThrowPositionOutOfRange(Int128 position, long maxPosition) => + throw new ArgumentOutOfRangeException("position", $"The position {position} is outside of [0, {maxPosition}]."); + + [DoesNotReturn] + public static void ThrowSeekOutOfRange(Int128 target, long maxPosition) => + throw new ArgumentOutOfRangeException("offset", $"The position {target} is outside of [0, {maxPosition}]."); +} From 6a6cf3748a5ab82d24cdc5dca3f0acb58b04f112 Mon Sep 17 00:00:00 2001 From: Marlon Date: Fri, 13 Nov 2026 02:24:06 +0100 Subject: [PATCH 2/3] test: cover the reader contract for corrupted data ReaderContractTests runs the same truncated, oversized and invalid inputs against every reader and checks the exception type and where Position ends up. ReaderSpecificTests covers what only one reader has, such as the file reader's cache and the stream reader's position restore. The round-trip tests now include non-ASCII text, and the file reader runs them with several cache sizes. InternalsVisibleTo lets the tests swap the file reader's buffer pool for one that fails on a double return. Co-Authored-By: Claude Opus 5.5 --- .../ReaderContractTests.cs | 481 ++++++++++++++ .../ReaderSpecificTests.cs | 609 ++++++++++++++++++ src/GenericReader.Tests/Tests.cs | 27 +- src/GenericReader/GenericReader.csproj | 4 + 4 files changed, 1113 insertions(+), 8 deletions(-) create mode 100644 src/GenericReader.Tests/ReaderContractTests.cs create mode 100644 src/GenericReader.Tests/ReaderSpecificTests.cs diff --git a/src/GenericReader.Tests/ReaderContractTests.cs b/src/GenericReader.Tests/ReaderContractTests.cs new file mode 100644 index 0000000..5f271da --- /dev/null +++ b/src/GenericReader.Tests/ReaderContractTests.cs @@ -0,0 +1,481 @@ +using System.Runtime.CompilerServices; +using System.Text; + +namespace GenericReader.Tests; + +public delegate void ReadAction(ref TReader reader) where TReader : IGenericReader, allows ref struct; + +// Every reader has to fail the same way for the same data: EndOfStreamException when the data runs out, +// InvalidDataException for a length in the data that cannot be valid, ArgumentOutOfRangeException for invalid +// arguments. A failed read does not move Position; only a length prefix that was read before the failure stays read. +public abstract class ReaderContractTests : IDisposable where TReader : IGenericReader, allows ref struct +{ + private readonly List _files = []; + + protected abstract TReader Create(byte[] data); + + protected string CreateFile(byte[] data) + { + string path = Path.GetTempFileName(); + _files.Add(path); + File.WriteAllBytes(path, data); + return path; + } + + public void Dispose() + { + foreach (string path in _files) + File.Delete(path); + } + + private void AssertThrows(byte[] data, long position, ReadAction read, long? positionAfter = null) + where TException : Exception + { + TReader reader = Create(data); + try + { + reader.PositionLong = position; + Exception? thrown = null; + try + { + read(ref reader); + } + catch (Exception ex) + { + thrown = ex; + } + + Assert.IsType(thrown); + Assert.Equal(positionAfter ?? position, reader.PositionLong); + } + finally + { + reader.Dispose(); + } + } + + // a valid read, and where it leaves Position + private void AssertReads(byte[] data, ReadAction read, long positionAfter, long position = 0) + { + TReader reader = Create(data); + try + { + reader.PositionLong = position; + read(ref reader); + Assert.Equal(positionAfter, reader.PositionLong); + } + finally + { + reader.Dispose(); + } + } + + private void AssertReadNearEnd() where T : unmanaged + { + int size = Unsafe.SizeOf(); + byte[] data = new byte[64]; + + for (int left = 0; left < size; left++) + AssertThrows(data, data.Length - left, (ref r) => r.Read()); + + TReader reader = Create(data); + try + { + reader.Position = data.Length - size; + reader.Read(); + Assert.Equal(data.Length, reader.Position); + } + finally + { + reader.Dispose(); + } + } + + [Fact] + public void ReadThrowsWhenFewerThanSizeOfTBytesAreLeft() + { + AssertReadNearEnd(); + AssertReadNearEnd(); + AssertReadNearEnd(); + AssertReadNearEnd(); + AssertReadNearEnd(); + AssertReadNearEnd(); + AssertReadNearEnd(); + } + + [Theory] + [InlineData(17)] + [InlineData(1000)] + [InlineData(int.MaxValue)] + public void ReadsPastTheEndThrow(int position) + { + byte[] data = new byte[16]; + AssertThrows(data, position, (ref r) => r.Read()); + AssertThrows(data, position, (ref r) => r.Read()); + AssertThrows(data, position, (ref r) => r.ReadArray(1)); + AssertThrows(data, position, (ref r) => r.ReadString(1, Encoding.UTF8)); + AssertThrows(data, position, (ref r) => r.ReadFString()); + AssertThrows(data, position, (ref r) => r.Read(new byte[1])); + // past the end, not even zero bytes can be read + AssertThrows(data, position, (ref r) => r.Read(Span.Empty)); + } + + [Fact] + public void ZeroLengthReadsPastTheEndThrow() + { + byte[] data = new byte[16]; + ReadAction[] reads = + [ + (ref r) => r.Read(Span.Empty), + (ref r) => r.ReadArray(0), + (ref r) => r.ReadString(0, Encoding.UTF8), + (ref r) => r.ReadFStringArray(0), + (ref r) => r.ReadArray(0, () => 0), + ]; + + foreach (ReadAction read in reads) + { + AssertThrows(data, 17, read); + // at the end, there are zero bytes to read + AssertReads(data, read, positionAfter: 16, position: 16); + } + } + + [Fact] + public void NullArgumentsThrow() + { + byte[] data = [.. TestBytes.Int32(4), .. new byte[12]]; + AssertThrows(data, 0, (ref r) => r.ReadString(null!)); + AssertThrows(data, 0, (ref r) => r.ReadString(4, null!)); + } + + [Fact] + public void ADecoderErrorDoesNotMovePosition() + { + var strict = new UTF8Encoding(false, throwOnInvalidBytes: true); + byte[] data = [.. TestBytes.Int32(1), 0xFF]; + AssertThrows(data, 4, (ref r) => r.ReadString(1, strict)); + AssertThrows(data, 0, (ref r) => r.ReadString(strict), positionAfter: 4); + } + + [Fact] + public void ReadsWithAnOffset() + { + byte[] data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + AssertReads(data, (ref r) => Assert.Equal(0x08070605, r.Read(4, SeekOrigin.Begin)), positionAfter: 8); + AssertReads(data, (ref r) => Assert.Equal(new byte[] { 11, 12 }, r.ReadArray(2, -2, SeekOrigin.End)), positionAfter: 12); + + // the seek is a step of its own: when the read after it fails, Position stays where the seek moved it + AssertThrows(data, 0, (ref r) => r.Read(10, SeekOrigin.Begin), positionAfter: 10); + // an invalid seek fails before anything moves + AssertThrows(data, 3, (ref r) => r.Read(-4, SeekOrigin.Current)); + } + + [Fact] + public void TruncatedReadsThrow() + { + byte[] data = new byte[16]; + + AssertThrows(data, 8, (ref r) => r.ReadArray(9)); + AssertThrows(data, 4, (ref r) => r.ReadArray(4)); + AssertThrows(data, 14, (ref r) => r.ReadBoolean()); + AssertThrows(data, 9, (ref r) => r.ReadString(8, Encoding.UTF8)); + // with a length prefix: the prefix stays read + AssertThrows([.. TestBytes.Int32(13), .. new byte[12]], 0, + (ref r) => r.ReadArray(), positionAfter: 4); + AssertThrows([.. TestBytes.Int32(13), .. new byte[12]], 0, + (ref r) => r.ReadString(Encoding.UTF8), positionAfter: 4); + } + + [Fact] + public void AFailedReadLeavesTheDestinationAlone() + { + byte[] data = [1, 2, 3, 4, 5, 6]; + int[] dest = [-1, -1]; + AssertThrows(data, 0, (ref r) => r.Read(dest)); + Assert.Equal(new[] { -1, -1 }, dest); + } + + [Fact] + public void ReadIntoADestination() + { + byte[] data = [1, 0, 0, 0, 2, 0, 0, 0]; + TReader reader = Create(data); + try + { + reader.Position = 3; + reader.Read(Span.Empty); + Assert.Equal(3, reader.Position); + reader.Position = data.Length; + reader.Read(Span.Empty); + Assert.Equal(data.Length, reader.Position); + + int[] dest = new int[2]; + reader.Position = 0; + reader.Read(dest); + Assert.Equal(new[] { 1, 2 }, dest); + Assert.Equal(data.Length, reader.Position); + } + finally + { + reader.Dispose(); + } + } + + [Fact] + public void ByteCountsThatOverflowThrow() + { + byte[] data = new byte[16]; + // 0x40000001 * 4 used to wrap around to 4 bytes + AssertThrows(data, 0, (ref r) => r.ReadArray(0x4000_0001)); + AssertThrows(data, 0, (ref r) => r.ReadArray(int.MaxValue)); + AssertThrows(data, 0, (ref r) => r.ReadArray(int.MaxValue)); + } + + [Fact] + public void ACorruptedLengthAllocatesNothing() + { + byte[] data = new byte[16]; + byte[] one = TestBytes.FString("abc", false); + + long allocated = GC.GetAllocatedBytesForCurrentThread(); + AssertThrows(data, 0, (ref r) => r.ReadArray(1 << 30)); + AssertThrows(data, 0, (ref r) => r.ReadString(1 << 30, Encoding.UTF8)); + AssertThrows([.. TestBytes.Int32(1 << 30), .. data], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + // int.MaxValue strings would have allocated a 16 GiB array before reading the first one + AssertThrows([.. TestBytes.Int32(int.MaxValue), .. one], 0, + (ref r) => r.ReadFStringArray(), positionAfter: 4); + Assert.True(GC.GetAllocatedBytesForCurrentThread() - allocated < 1 << 20); + } + + [Fact] + public void NegativeArgumentsThrow() + { + byte[] data = new byte[16]; + AssertThrows(data, 0, (ref r) => r.ReadArray(-1)); + AssertThrows(data, 0, (ref r) => r.ReadArray(int.MinValue)); + AssertThrows(data, 0, (ref r) => r.ReadString(-1, Encoding.UTF8)); + AssertThrows(data, 0, (ref r) => r.ReadFStringArray(-1)); + AssertThrows(data, 0, (ref r) => r.ReadArray(-1, () => 0)); + } + + [Fact] + public void NegativeLengthPrefixesAreInvalidData() + { + byte[] data = [.. TestBytes.Int32(-1), .. new byte[12]]; + AssertThrows(data, 0, (ref r) => r.ReadArray(), positionAfter: 4); + AssertThrows(data, 0, (ref r) => r.ReadString(Encoding.UTF8), positionAfter: 4); + AssertThrows(data, 0, (ref r) => r.ReadFStringArray(), positionAfter: 4); + AssertThrows(data, 0, (ref r) => r.ReadArray(() => 0), positionAfter: 4); + } + + [Fact] + public void ReadFStringAtTheEndOfTheData() + { + // not even the length + AssertThrows([4, 0, 0], 0, (ref r) => r.ReadFString()); + // UTF-8: one byte missing + AssertThrows(TestBytes.FString("abc", false)[..^1], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + // UTF-16: the characters are there but the terminator is not + AssertThrows(TestBytes.FString("abc", true)[..^2], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + AssertThrows(TestBytes.FString("abc", true)[..^1], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + // UTF-16 empty string without its terminator + AssertThrows(TestBytes.Int32(-1), 0, (ref r) => r.ReadFString(), positionAfter: 4); + + // complete strings that end exactly at the end of the data + foreach (bool unicode in new[] { false, true }) + { + byte[] data = TestBytes.FString("abc", unicode); + TReader reader = Create(data); + try + { + Assert.Equal("abc", reader.ReadFString()); + Assert.Equal(data.Length, reader.Position); + } + finally + { + reader.Dispose(); + } + } + } + + [Theory] + [InlineData(int.MinValue)] + [InlineData(-0x4000_0000)] + public void FStringLengthsWhoseByteCountDoesNotFitAreInvalidData(int length) + { + AssertThrows([.. TestBytes.Int32(length), .. new byte[16]], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + } + + [Fact] + public void TheLargestUtf16FStringLengthIsOnlyShortOfData() + { + // -0x3FFFFFFF chars are 0x7FFFFFFE bytes, which fit in an int: this data is merely too short + AssertThrows([.. TestBytes.Int32(-0x3FFF_FFFF), .. new byte[16]], 0, + (ref r) => r.ReadFString(), positionAfter: 4); + } + + [Fact] + public void ReadFStringArray() + { + byte[] one = TestBytes.FString("abc", false); + + // a count that fits the data size but runs out of strings + byte[] two = [.. TestBytes.Int32(2), .. one]; + AssertThrows(two, 0, (ref r) => r.ReadFStringArray(), positionAfter: two.Length); + + byte[] both = [.. TestBytes.Int32(2), .. one, .. TestBytes.FString("de", true)]; + TReader reader = Create(both); + try + { + Assert.Equal(new[] { "abc", "de" }, reader.ReadFStringArray()); + Assert.Equal(both.Length, reader.Position); + reader.Position = 0; + Assert.Empty(reader.ReadFStringArray(0)); + } + finally + { + reader.Dispose(); + } + } + + // the exception that read throws, or null; a lambda cannot capture a ref struct, so the reader is passed by ref + private static Exception? Record(ref TReader reader, ReadAction read) + { + try + { + read(ref reader); + return null; + } + catch (Exception ex) + { + return ex; + } + } + + [Fact] + public void GetterArraysAreLimitedToMaxArrayAllocationSize() + { + byte[] data = new byte[16]; + TReader reader = Create(data); + try + { + Assert.Equal(IGenericReader.DefaultMaxArrayAllocationSize, reader.MaxArrayAllocationSize); + + // rejected before the array is allocated or the getter runs + int calls = 0; + long allocated = GC.GetAllocatedBytesForCurrentThread(); + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray(int.MaxValue, () => ++calls))); + // the limit is in bytes: 2^24 elements of 256 bytes used to allocate 4 GiB from a 4-byte count + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray(1 << 24, () => default(Big256)))); + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray(1 << 26, () => ""))); + Assert.True(GC.GetAllocatedBytesForCurrentThread() - allocated < 1 << 20); + Assert.Equal(0, calls); + + reader.MaxArrayAllocationSize = 3 * sizeof(int); + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray(4, () => ++calls))); + Assert.Equal(0, calls); + Assert.Equal(new[] { 1, 2, 3 }, reader.ReadArray(3, () => ++calls)); + + // zero-byte elements: the limit is the only bound + reader.MaxArrayAllocationSize = 100 * sizeof(int); + Assert.Equal(100, reader.ReadArray(100, () => 0).Length); + Assert.Equal(0, reader.Position); + + Assert.IsType(Record(ref reader, (ref r) => r.MaxArrayAllocationSize = -1)); + Assert.Equal(100 * sizeof(int), reader.MaxArrayAllocationSize); + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray(1, (Func)null!))); + Assert.IsType(Record(ref reader, (ref r) => r.ReadArray((Func)null!))); + Assert.Equal(0, reader.Position); + } + finally + { + reader.Dispose(); + } + + // with a length prefix + AssertThrows([.. TestBytes.Int32(int.MaxValue), .. data], 0, + (ref r) => r.ReadArray(() => 0), positionAfter: 4); + } + + [Fact] + public void InvalidPositionsThrow() + { + TReader reader = Create(new byte[16]); + try + { + reader.Position = 5; + ReadAction[] invalid = + [ + (ref r) => r.Position = -1, + (ref r) => r.PositionLong = long.MinValue, + (ref r) => r.SetPosition(Int128.MinValue), + (ref r) => r.SetPosition(UInt128.MaxValue), + (ref r) => r.Seek(-6, SeekOrigin.Current), + (ref r) => r.Seek(-1, SeekOrigin.Begin), + (ref r) => r.Seek(-17, SeekOrigin.End), + // these used to wrap around to a negative or a small position + (ref r) => r.SeekLong(long.MaxValue, SeekOrigin.Current), + (ref r) => r.SeekLong(UInt128.MaxValue, SeekOrigin.Current), + (ref r) => r.SeekLong(Int128.MinValue, SeekOrigin.End), + (ref r) => r.Seek(0, (SeekOrigin)3), + // the int overload cannot return this; it used to move Position first on the file and stream readers + (ref r) => r.Seek(1L << 31, SeekOrigin.Begin), + ]; + + foreach (ReadAction action in invalid) + { + Assert.IsType(Record(ref reader, action)); + Assert.Equal(5, reader.Position); + } + + // like a stream, a position past the end is allowed, only reading from it fails + Assert.Equal(100, reader.Seek(95)); + Assert.Equal(16, reader.Seek(0, SeekOrigin.End)); + Assert.Equal(2, reader.Seek(-14, SeekOrigin.Current)); + reader.SetPosition(7u); + Assert.Equal(7, reader.GetPosition()); + } + finally + { + reader.Dispose(); + } + } +} + +[InlineArray(256)] +public struct Big256 +{ + private byte _element; +} + +public class SpanReaderContractTests : ReaderContractTests +{ + protected override GenericSpanReader Create(byte[] data) => new(data); +} + +public class BufferReaderContractTests : ReaderContractTests +{ + protected override GenericBufferReader Create(byte[] data) => new(data); +} + +public class StreamReaderContractTests : ReaderContractTests +{ + protected override GenericStreamReader Create(byte[] data) => new(new MemoryStream(data)); +} + +public class FileReaderContractTests : ReaderContractTests +{ + protected override GenericFileReader Create(byte[] data) => new(CreateFile(data)); +} + +// a cache smaller than most reads, so that they go straight to the file +public class SmallBufferFileReaderContractTests : ReaderContractTests +{ + protected override GenericFileReader Create(byte[] data) => new(CreateFile(data), bufferSize: 3); +} diff --git a/src/GenericReader.Tests/ReaderSpecificTests.cs b/src/GenericReader.Tests/ReaderSpecificTests.cs new file mode 100644 index 0000000..cdf0c82 --- /dev/null +++ b/src/GenericReader.Tests/ReaderSpecificTests.cs @@ -0,0 +1,609 @@ +using System.Buffers; +using System.Text; + +using Microsoft.Win32.SafeHandles; + +namespace GenericReader.Tests; + +public class SpanReaderSpecificTests +{ + [Fact] + public void ReadDoesNotReadPastASlice() + { + // the reader sees the first 8 bytes; the old check let Read at 6 take two of the 0xFF bytes + byte[] buffer = [1, 2, 3, 4, 5, 6, 7, 8, 0xFF, 0xFF, 0xFF, 0xFF]; + var reader = new GenericSpanReader(buffer, 0, 8) { Position = 6 }; + Assert.Equal(0x0807, reader.Read()); + Assert.Throws(() => new GenericSpanReader(buffer, 0, 8) { Position = 6 }.Read()); + } + + [Fact] + public void ReadSpanStaysInsideTheData() + { + byte[] data = new byte[16]; + var reader = new GenericSpanReader(data) { Position = 12 }; + + // 0x40000001 * 4 wrapped around to 4 bytes, yet the span came back with 0x40000001 elements + Assert.Throws(() => new GenericSpanReader(data).ReadSpan(0x4000_0001)); + Assert.Throws(() => new GenericSpanReader(data) { Position = 12 }.ReadSpan(2)); + Assert.Throws(() => new GenericSpanReader(data) { Position = 12 }.ReadSpan(5)); + Assert.Throws(() => new GenericSpanReader(data).ReadSpan(-1)); + Assert.Throws(() => new GenericSpanReader(data).ReadSpan(-0x4000_0000)); + Assert.Throws(() => new GenericSpanReader(TestBytes.Int32(-1)).ReadSpan()); + + Assert.Equal(1, reader.ReadSpan(1).Length); + Assert.Equal(16, reader.Position); + Assert.Equal(0, reader.ReadSpan(0).Length); + } + + [Fact] + public void GetterFuncArraysAreLimitedToMaxArrayAllocationSize() + { + byte[] data = [1, 2, 3, 4]; + var reader = new GenericSpanReader(data) { MaxArrayAllocationSize = 3 }; + + Assert.Equal(new byte[] { 1, 2, 3 }, reader.ReadArray(3, static (ref r) => r.Read())); + Assert.Throws(() => + new GenericSpanReader(data) { MaxArrayAllocationSize = 3 }.ReadArray(4, static (ref r) => r.Read())); + } + + [Fact] + public void ADefaultReaderHasTheDefaultLimit() + { + // the limit used to be 0 here, so that every getter array failed + GenericSpanReader reader = default; + Assert.Equal(IGenericReader.DefaultMaxArrayAllocationSize, reader.MaxArrayAllocationSize); + Assert.Equal(0, reader.Length); + } + + [Fact] + public void SlicesKeepMaxArrayAllocationSize() + { + var reader = new GenericSpanReader(new byte[16]) { MaxArrayAllocationSize = 5, Position = 4 }; + Assert.Equal(5, reader.Slice(2).MaxArrayAllocationSize); + Assert.Equal(5, reader.Slice(2, 4, sliceAtPosition: true).MaxArrayAllocationSize); + } + + [Fact] + public void ReadsReadOnlyData() + { + // read-only memory (here in the assembly image) used to come back as writable spans + var reader = new GenericSpanReader("abcd"u8); + ReadOnlySpan span = reader.ReadSpan(2); + Assert.Equal((byte)'a', span[0]); + Assert.Equal("cd", reader.ReadString(2, Encoding.UTF8)); + } + + [Fact] + public void GettersTakingIGenericReaderThrowThroughTheInterface() + { + // [Obsolete(error: true)] does not stop a call through a generic TReader; these returned an uninitialized + // array without calling the getter + Assert.Throws(() => ReadThroughInterface(new GenericSpanReader(new byte[16]))); + } + + private static int[] ReadThroughInterface(TReader reader) where TReader : IGenericReader, allows ref struct + { + return reader.ReadArray(4, static r => r.Read()); + } +} + +public class BufferReaderSpecificTests +{ + [Fact] + public void ReadsDoNotLeaveTheMemory() + { + // the reader sees the first 8 bytes; the old checks let reads take the 0xFF bytes behind them + byte[] buffer = [1, 2, 3, 4, 5, 6, 7, 8, 0xFF, 0xFF, 0xFF, 0xFF]; + var reader = new GenericBufferReader(buffer, 0, 8) { Position = 6 }; + + Assert.Throws(() => reader.Read()); + Assert.Throws(() => reader.ReadArray(3)); + Assert.Throws(() => reader.ReadSpan(0x4000_0001)); + Assert.Throws(() => reader.ReadSpan(3)); + Assert.Throws(() => reader.ReadMemory(3)); + Assert.Throws(() => reader.ReadMemory(-1)); + Assert.Equal(6, reader.Position); + + Assert.Equal(new byte[] { 7, 8 }, reader.ReadMemory(2).ToArray()); + Assert.Equal(8, reader.Position); + } + + [Fact] + public void ReadFStringMemory() + { + // UTF-16 without its terminator used to succeed and leave Position past the end + var reader = new GenericBufferReader(TestBytes.FString("abc", true)[..^2]); + Assert.Throws(() => reader.ReadFStringMemory()); + Assert.Equal(4, reader.Position); + + reader = new GenericBufferReader([.. TestBytes.Int32(int.MinValue), .. new byte[16]]); + Assert.Throws(() => reader.ReadFStringMemory()); + + foreach (bool unicode in new[] { false, true }) + { + byte[] data = TestBytes.FString("abc", unicode); + reader = new GenericBufferReader(data); + FStringMemory value = reader.ReadFStringMemory(); + Assert.Equal("abc", value.ToString()); + Assert.Equal(unicode, value.IsUnicode); + Assert.Equal(data.Length, reader.Position); + } + } + + [Fact] + public void ReadFStringMemoryArrayWithACountBeyondTheData() + { + byte[] data = TestBytes.FString("abc", true); + + // int.MaxValue strings would have allocated the array before reading the first one + var reader = new GenericBufferReader([.. TestBytes.Int32(int.MaxValue), .. data]); + long allocated = GC.GetAllocatedBytesForCurrentThread(); + Assert.Throws(() => reader.ReadFStringMemoryArray()); + Assert.True(GC.GetAllocatedBytesForCurrentThread() - allocated < 1 << 20); + Assert.Equal(4, reader.Position); + + Assert.Throws(() => reader.ReadFStringMemoryArray(-1)); + Assert.Throws(() => new GenericBufferReader(TestBytes.Int32(-1)).ReadFStringMemoryArray()); + + reader = new GenericBufferReader([.. TestBytes.Int32(1), .. data]); + Assert.Equal("abc", Assert.Single(reader.ReadFStringMemoryArray()).ToString()); + } + + [Fact] + public void SlicesKeepMaxArrayAllocationSize() + { + var reader = new GenericBufferReader(new byte[16]) { MaxArrayAllocationSize = 5 }; + Assert.Equal(5, reader.Slice(2).MaxArrayAllocationSize); + Assert.Equal(5, reader.Slice(2, 4).MaxArrayAllocationSize); + } + + [Fact] + public void ReadMemory() + { + var reader = new GenericBufferReader([.. TestBytes.Int32(3), 7, 8, 9, 10]); + Assert.Equal(new byte[] { 7, 8, 9 }, reader.ReadMemory().ToArray()); + Assert.Equal(new byte[] { 10 }, reader.AsMemory(sliceAtPosition: true).ToArray()); + Assert.Equal(8L, reader.GetLength()); + } + + [Fact] + public async Task LoadFromStreamReadsTheRestOfTheStream() + { + // the stream is not at its start: the loaders used to allocate its whole length and then run out of data + byte[] data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var stream = new MemoryStream(data) { Position = 4 }; + using (GenericBufferReader reader = GenericBufferReader.LoadFromStream(stream)) + Assert.Equal(data[4..], reader.AsSpan().ToArray()); + + stream.Position = 4; + using (GenericBufferReader reader = await GenericBufferReader.LoadFromStreamAsync(stream, TestContext.Current.CancellationToken)) + Assert.Equal(data[4..], reader.AsSpan().ToArray()); + } + + [Fact] + public async Task LoadFromFile() + { + string path = Path.GetTempFileName(); + try + { + byte[] data = new byte[100_000]; + Random.Shared.NextBytes(data); + await File.WriteAllBytesAsync(path, data, TestContext.Current.CancellationToken); + + using (SafeFileHandle handle = File.OpenHandle(path)) + using (GenericBufferReader reader = GenericBufferReader.LoadFromFile(handle)) + Assert.Equal(data, reader.AsSpan().ToArray()); + + using (SafeFileHandle handle = File.OpenHandle(path, options: FileOptions.Asynchronous)) + using (GenericBufferReader reader = await GenericBufferReader.LoadFromFileAsync(handle, TestContext.Current.CancellationToken)) + Assert.Equal(data, reader.AsSpan().ToArray()); + } + finally + { + File.Delete(path); + } + } +} + +public class FileReaderSpecificTests : IDisposable +{ + private readonly string _filePath = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_filePath); + } + + [Fact] + public void ReadsDoNotSeeTheRestOfThePooledArray() + { + // leave a dirty array where the reader rents its buffer from: the old reader handed the whole rented array + // to its inner reader, so reads past the end of the file returned these bytes instead of throwing + byte[] dirty = ArrayPool.Shared.Rent(4096); + dirty.AsSpan().Fill(0xCC); + ArrayPool.Shared.Return(dirty); + + File.WriteAllBytes(_filePath, [1, 2, 3, 4, 5, 6]); + using var reader = new GenericFileReader(_filePath); + + Assert.Throws(() => reader.Read()); + Assert.Equal(0x04030201, reader.Read()); + Assert.Throws(() => reader.Read()); + Assert.Throws(() => reader.ReadBytes(3)); + Assert.Equal(4, reader.Position); + Assert.Equal(0x0605, reader.Read()); + Assert.Empty(reader.ReadBytes(0)); + Assert.Equal(6, reader.Position); + } + + [Fact] + public void ReadBytes() + { + byte[] data = new byte[10_000]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(_filePath, data); + using var reader = new GenericFileReader(_filePath); + + long allocated = GC.GetAllocatedBytesForCurrentThread(); + Assert.Throws(() => reader.ReadBytes(1 << 30)); + Assert.Throws(() => reader.ReadBytes(1 << 30, useSharedArrayPool: true)); + Assert.True(GC.GetAllocatedBytesForCurrentThread() - allocated < 1 << 20); + Assert.Throws(() => reader.ReadBytes(-1)); + + // from the cache, and larger than it + Assert.Equal(data[..100], reader.ReadBytes(100)); + Assert.Equal(data[100..9000], reader.ReadBytes(8900)); + byte[] pooled = reader.ReadBytes(1000, useSharedArrayPool: true); + Assert.Equal(data[9000..], pooled[..1000]); + ArrayPool.Shared.Return(pooled); + Assert.Equal(data.Length, reader.Position); + } + + [Fact] + public void ALargeReadDoesNotBreakTheCache() + { + byte[] data = new byte[64 * 1024]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(_filePath, data); + using var reader = new GenericFileReader(_filePath, bufferSize: 256); + + Assert.Equal(data[..1000], reader.ReadArray(1000)); + Assert.Equal(Encoding.UTF8.GetString(data, 1000, 48 * 1024), reader.ReadString(48 * 1024, Encoding.UTF8)); + + // small reads around and across the cache end, and backwards + foreach (int position in new[] { 0, 252, 253, 255, 256, 1000, 50_000, 3, 65_532 }) + { + reader.Position = position; + Assert.Equal(BitConverter.ToInt32(data, position), reader.Read()); + Assert.Equal(position + 4, reader.Position); + } + } + + [Fact] + public void AFileThatGrowsIsOnlyReadUpToItsLength() + { + File.WriteAllBytes(_filePath, new byte[100]); + using var reader = new GenericFileReader(OpenShared()); + + using (var writer = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) + writer.Write(new byte[100]); + + // the cache used to fill past Length, so that the bytes behind it could be read from the cache + reader.Position = 96; + reader.Read(); + Assert.Throws(() => reader.Read()); + reader.Position = 50; + reader.Read(); + reader.Position = 150; + Assert.Throws(() => reader.Read()); + Assert.Equal(100, reader.Length); + } + + [Fact] + public void AHugePositionDoesNotWrapAroundIntoTheCache() + { + File.WriteAllBytes(_filePath, new byte[32]); + using var reader = new GenericFileReader(_filePath); + reader.Read(); // the cache holds the whole file + + // position + size used to overflow and pass the cache check, which then read at a garbage offset + foreach (long position in new[] { long.MaxValue, long.MaxValue - 1, long.MaxValue - 15, 1L << 32 }) + { + reader.PositionLong = position; + Assert.Throws(() => reader.Read()); + Assert.Throws(() => reader.ReadArray(16)); + Assert.Equal(position, reader.PositionLong); + } + } + + [Fact] + public void AFileThatGetsShorterThrows() + { + File.WriteAllBytes(_filePath, new byte[16]); + using var cached = new GenericFileReader(OpenShared()); + using var direct = new GenericFileReader(OpenShared(), bufferSize: 1); + + using (var writer = new FileStream(_filePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) + writer.SetLength(6); + + // through the cache, and past it + Assert.Throws(() => cached.Read()); + Assert.Equal(0, cached.Position); + Assert.Throws(() => direct.Read()); + Assert.Equal(0, direct.Position); + } + + private SafeFileHandle OpenShared() => + File.OpenHandle(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, FileOptions.RandomAccess); + + [Fact] + public void DisposeIsIdempotentAndStopsReads() + { + File.WriteAllBytes(_filePath, new byte[10_000]); + var pool = new CountingPool(); + var reader = new GenericFileReader(_filePath) { BufferPool = pool }; + reader.Read(); + reader.Position = 9000; // a cache miss refills the same array instead of renting another + reader.Read(); + Assert.Equal(1, pool.Rented); + + // a second Dispose used to return the cache array to the pool again + reader.Dispose(); + reader.Dispose(); + Assert.Equal(1, pool.Returned); + + // the cache used to keep serving the array that went back to the pool + Assert.Throws(() => reader.Read()); + Assert.Throws(() => reader.ReadBytes(8000)); + Assert.Equal(1, pool.Rented); + } + + // Counts rents and returns, and fails on returning an array that is not rented (a double return). + private sealed class CountingPool : ArrayPool + { + private readonly HashSet _rented = new(ReferenceEqualityComparer.Instance); + + public int Rented { get; private set; } + public int Returned { get; private set; } + + public override byte[] Rent(int minimumLength) + { + byte[] array = new byte[minimumLength]; + _rented.Add(array); + Rented++; + return array; + } + + public override void Return(byte[] array, bool clearArray = false) + { + Assert.True(_rented.Remove(array), "The array was returned twice or was not rented here."); + Returned++; + } + } + + [Fact] + public void BufferSizeHasToBePositive() + { + File.WriteAllBytes(_filePath, new byte[16]); + Assert.Throws(() => new GenericFileReader(_filePath, bufferSize: 0)); + Assert.Throws(() => new GenericFileReader(_filePath, bufferSize: -1)); + + // the rejected readers left no open handle behind + File.Delete(_filePath); + } + + [Fact] + public void LeaveOpenKeepsTheHandle() + { + File.WriteAllBytes(_filePath, [1, 2, 3, 4]); + using SafeFileHandle handle = OpenShared(); + + using (var reader = new GenericFileReader(handle, leaveOpen: true)) + Assert.Equal(0x04030201, reader.Read()); + Assert.False(handle.IsClosed); + + using (GenericFileReader reader = IGenericReader.Create(handle, leaveOpen: true)) + reader.Read(); + Assert.False(handle.IsClosed); + + new GenericFileReader(handle).Dispose(); + Assert.True(handle.IsClosed); + } +} + +public class StreamReaderSpecificTests +{ + [Fact] + public void ReadBytes() + { + byte[] data = new byte[16]; + Random.Shared.NextBytes(data); + using var reader = new GenericStreamReader(new MemoryStream(data)); + + long allocated = GC.GetAllocatedBytesForCurrentThread(); + Assert.Throws(() => reader.ReadBytes(1 << 30)); + Assert.Throws(() => reader.ReadBytes(1 << 30, useSharedArrayPool: true)); + Assert.True(GC.GetAllocatedBytesForCurrentThread() - allocated < 1 << 20); + Assert.Throws(() => reader.ReadBytes(-1)); + Assert.Equal(0, reader.Position); + + Assert.Equal(data[..10], reader.ReadBytes(10)); + byte[] pooled = reader.ReadBytes(6, useSharedArrayPool: true); + Assert.Equal(data[10..], pooled[..6]); + ArrayPool.Shared.Return(pooled); + } + + [Fact] + public void AStreamThatIsNotAtItsStart() + { + byte[] data = [1, 2, 3, 4, 5, 6, 7, 8]; + using var reader = new GenericStreamReader(new MemoryStream(data) { Position = 4 }); + Assert.Equal(4, reader.Position); + Assert.Equal(8, reader.Length); + Assert.Equal(0x08070605, reader.Read()); + Assert.Throws(() => reader.Read()); + } + + [Fact] + public void AStreamThatEndsBeforeItsLengthPutsPositionBack() + { + byte[] data = [.. TestBytes.Int32(8), 1, 2, 3, 4]; + using var reader = new GenericStreamReader(new ShortStream(data, reportedLength: 16)); + + Assert.Throws(() => reader.Read(4, SeekOrigin.Begin)); + Assert.Equal(4, reader.Position); + Assert.Throws(() => reader.ReadArray(8)); + Assert.Equal(4, reader.Position); + + reader.Position = 0; + Assert.Throws(() => reader.ReadFString()); + Assert.Equal(4, reader.Position); // after the length prefix, like the other readers + } + + // a stream whose Length claims more data than it holds + private sealed class ShortStream(byte[] data, long reportedLength) : MemoryStream(data) + { + public override long Length => reportedLength; + } + + [Fact] + public void LongStringsGoThroughThePool() + { + string value = new('x', 5000); + byte[] data = [.. TestBytes.FString(value, false), .. TestBytes.FString(value, true)]; + using var reader = new GenericStreamReader(new MemoryStream(data)); + + Assert.Equal(value, reader.ReadFString()); + Assert.Equal(value, reader.ReadFString()); + Assert.Equal(data.Length, reader.Position); + } + + [Fact] + public void LeaveOpenKeepsTheStream() + { + var stream = new MemoryStream([1, 2, 3, 4]); + + using (var reader = new GenericStreamReader(stream, leaveOpen: true)) + Assert.Equal(0x04030201, reader.Read()); + Assert.True(stream.CanRead); + + using (GenericStreamReader reader = IGenericReader.Create(stream, leaveOpen: true)) + Assert.Equal(4, reader.Position); + Assert.True(stream.CanRead); + + new GenericStreamReader(stream).Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void AnOwnedStreamIsDisposedWhenTheConstructorFails() + { + // a stream that cannot seek has no Length + var stream = new UnseekableStream(); + Assert.Throws(() => new GenericStreamReader(stream)); + Assert.True(stream.Disposed); + + stream = new UnseekableStream(); + Assert.Throws(() => new GenericStreamReader(stream, leaveOpen: true)); + Assert.False(stream.Disposed); + } + + private sealed class UnseekableStream : Stream + { + public bool Disposed { get; private set; } + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => 0; + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + } +} + +// The readers that are classes behave the same after Dispose. The span reader owns nothing, so its Dispose does nothing. +public class DisposedReaderTests : IDisposable +{ + private readonly string _filePath = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_filePath); + } + + private GenericReaderBase Create(string kind, byte[] data) + { + File.WriteAllBytes(_filePath, data); + return kind switch + { + "buffer" => new GenericBufferReader(data), + "file" => new GenericFileReader(_filePath), + "stream" => new GenericStreamReader(new MemoryStream(data)), + // the stream stays usable, the reader does not + "stream, leaveOpen" => new GenericStreamReader(new MemoryStream(data), leaveOpen: true), + _ => throw new ArgumentOutOfRangeException(nameof(kind)) + }; + } + + [Theory] + [InlineData("buffer")] + [InlineData("file")] + [InlineData("stream")] + [InlineData("stream, leaveOpen")] + public void EveryReadAndPositionThrowsAfterDispose(string kind) + { + GenericReaderBase reader = Create(kind, [.. TestBytes.FString("abc", false), .. new byte[8]]); + Assert.Equal("abc", reader.ReadFString()); + reader.Dispose(); + reader.Dispose(); // a second Dispose does nothing + + Action[] actions = + [ + () => reader.Read(), + () => reader.Read(Span.Empty), + () => reader.ReadArray(0), + () => reader.ReadArray(4), + () => reader.ReadString(0, Encoding.UTF8), + () => reader.ReadFString(), + () => reader.ReadFStringArray(0), + () => reader.ReadArray(0, () => 0), + () => reader.ReadBoolean(), + () => _ = reader.Position, + () => _ = reader.PositionLong, + () => reader.Position = 0, + () => reader.SetPosition(0), + () => reader.GetPosition(), + () => reader.Seek(0, SeekOrigin.Begin), + () => reader.SeekLong(0), + () => reader.Read(0, SeekOrigin.Begin), + ]; + + foreach (Action action in actions) + Assert.Throws(action); + + // the length is still known + Assert.Equal(16, reader.Length); + } +} + +internal static class TestBytes +{ + public static byte[] Int32(int value) => BitConverter.GetBytes(value); + + // an FString: int32 length including the terminator, positive for UTF-8 bytes, negative for UTF-16 chars + public static byte[] FString(string value, bool unicode) + { + byte[] chars = (unicode ? Encoding.Unicode : Encoding.UTF8).GetBytes(value + '\0'); + int length = unicode ? -(value.Length + 1) : chars.Length; + return [.. Int32(length), .. chars]; + } +} diff --git a/src/GenericReader.Tests/Tests.cs b/src/GenericReader.Tests/Tests.cs index 472ca43..304fc79 100644 --- a/src/GenericReader.Tests/Tests.cs +++ b/src/GenericReader.Tests/Tests.cs @@ -1,4 +1,5 @@ -using System.Runtime.CompilerServices; +using System.Buffers; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; @@ -68,10 +69,15 @@ public FileReaderTests(BinaryDataGenerator gen) File.WriteAllBytes(_filePath, _gen.Data); } - [Fact] - public void ReadAll() + // small caches make reads straddle the cache end, and reads larger than the cache go straight to the file + [Theory] + [InlineData(4096)] + [InlineData(64)] + [InlineData(7)] + [InlineData(1)] + public void ReadAll(int bufferSize) { - var reader = new GenericFileReader(_filePath); + var reader = new GenericFileReader(_filePath, bufferSize); try { @@ -120,7 +126,7 @@ public void Test(ref TReader reader) Assert.True(verifyOperation(ref reader)); } - Assert.Equal(reader.Position, reader.Length); + Assert.Equal(reader.Length, reader.Position); } } @@ -169,6 +175,11 @@ public BinaryDataValues() AddFString(true, string.Empty); AddFString(true, Helpers.GetString()); AddFString(true, Helpers.GetString(128)); + // non-ASCII text, with surrogate pairs and multi-byte UTF-8 + AddString(Encoding.UTF8, "Grüße, Ελληνικά, 日本語, 😀"); + AddString(Encoding.Unicode, "Grüße, Ελληνικά, 日本語, 😀"); + AddFString(false, "Grüße, Ελληνικά, 日本語, 😀"); + AddFString(true, "Grüße, Ελληνικά, 日本語, 😀"); for (int i = 0; i < 32; i++) { Add(); @@ -201,7 +212,7 @@ private void Add() Unsafe.SkipInit(out TValue value); Helpers.Fill(ref value); Values.Add(value); - VerifyOperations.Add((ref TReader reader) => + VerifyOperations.Add((ref reader) => { TValue readValue = reader.Read(); return readValue.Equals(value); @@ -216,7 +227,7 @@ private void AddString(Encoding encoding, string? value = null) { value ??= Helpers.GetName(); Values.Add(value); - VerifyOperations.Add((ref TReader reader) => + VerifyOperations.Add((ref reader) => { string readValue = reader.ReadString(encoding); return readValue.Equals(value, StringComparison.Ordinal); @@ -235,7 +246,7 @@ private void AddFString(bool unicode, string? value = null, bool nullTerminated { value ??= Helpers.GetName(); Values.Add(value); - VerifyOperations.Add((ref TReader reader) => + VerifyOperations.Add((ref reader) => { string readValue = reader.ReadFString(); return readValue.Equals(value, StringComparison.Ordinal); diff --git a/src/GenericReader/GenericReader.csproj b/src/GenericReader/GenericReader.csproj index 6cd5d84..509872c 100644 --- a/src/GenericReader/GenericReader.csproj +++ b/src/GenericReader/GenericReader.csproj @@ -23,4 +23,8 @@ + + + + From f5a85b0546213b657e94affe834827ef27e8ed76 Mon Sep 17 00:00:00 2001 From: Marlon Date: Fri, 13 Nov 2026 02:24:07 +0100 Subject: [PATCH 3/3] docs: describe the error contract for corrupted data Co-Authored-By: Claude Opus 5.5 --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 647d772..55f45a4 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,32 @@ var numberFromSpan = spanReader.Read(); --- +## 🛡️ Corrupted Data + +Every reader checks each read against the data before it reads or allocates anything, and all readers fail the same way: + +| Exception | When | +|---|---| +| `EndOfStreamException` | The data ends before the read does, or the read starts past the end (even when it reads zero bytes) | +| `InvalidDataException` | The data contains a length that cannot be valid: a negative length prefix, an FString length whose byte count does not fit in an `int`, or a getter array larger than `MaxArrayAllocationSize` | +| `ArgumentOutOfRangeException` | An argument is invalid, such as a negative length, a position or seek target outside of `[0, max]`, or a negative `MaxArrayAllocationSize` | + +A read that fails does not move `Position`, with two exceptions: + +- A length prefix that was read before the failure stays read. +- Reads made of several steps keep the steps that succeeded. The overloads with an offset keep their seek, and an array of FStrings or of getter elements keeps the elements it read. + +Positions past the end are allowed, as with streams; reading there fails. After `Dispose`, the buffer, file and stream readers throw `ObjectDisposedException` for every read and every access to the position; `Length` stays available. Readers over memory return `ReadOnlySpan` and `ReadOnlyMemory` views of it. If a file or stream turns out shorter than the `Length` it had when the reader was created, the read throws `EndOfStreamException`, and the destination may hold part of the data. + +`ReadArray` with a getter allocates the array before it reads any element, and an element may take no bytes at all, so its length cannot be checked against the data. `MaxArrayAllocationSize` limits the array size in bytes instead. The default is `IGenericReader.DefaultMaxArrayAllocationSize`, 256 MiB: + +```csharp +reader.MaxArrayAllocationSize = 64 * 1024 * 1024; +var entries = reader.ReadArray(() => new Entry(reader)); +``` + +--- + ## 🤝 Contributing Contributions are **welcome and appreciated**!