Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ API between minor releases.
- The position and count of `ObjectDirectoryRecord` are now documented: a byte extent of the decoded global stream,
confirmed against the object directories of real files.

- `PlaDocument.FindCompressedPools()` finds pools outside indexed block payloads. `CompressedPool.Decode()` reads
stored and QuickLZ level 1 frames using the public format description and saved-file verification.

### Changed

- The release workflow follows the IOExtend release order: the NuGet packages are pushed before the GitHub release is
Expand Down
14 changes: 14 additions & 0 deletions docs/public-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,20 @@ The `PlaIO` package is the supported surface listed here; every public type is i
| `PlaStream.ReadExtent(long position, long count)` | The bytes of one extent. Throws `ArgumentOutOfRangeException` for negative or out-of-bounds extents and `InvalidDataException` for extents that reach the unreliable range. |
| `PlaStream.ReadExtent(ObjectDirectoryRecord record)` | The bytes addressed by a directory record. Throws `InvalidDataException` if the extent exceeds the stream or reaches the unreliable range. |

## Compressed pools

Pool framing was verified using saved container bytes. A 16-byte marker is followed by the decoded total and frame count
as little-endian u64 values. Each frame has a u32 size followed by a standalone QuickLZ level 1 frame. The decoder
implements the [public QuickLZ 1.5.0 format](https://github.com/ReSpeak/quicklz/blob/d905415a88aed6dae685bae500025ccc81de3822/Format.md);
independent decoding of saved-file frames verified the correspondence. PlaIO has no runtime dependency on QuickLZ.

| Member | Use |
|---|---|
| `PlaDocument.FindCompressedPools()` | Finds pool markers outside indexed block payloads and validates frame headers and sizes. Computed on each call. |
| `CompressedPool.Offset` and `End` | File offsets of the pool marker and of the end of the pool. |
| `CompressedPool.TotalDecompressed` and `BlockCount` | Decoded byte count and number of frames in the pool. |
| `CompressedPool.Decode()` | Decoded frame payloads in order. Throws `InvalidDataException` on invalid framing or data and `ObjectDisposedException` after disposing the document. |

## BIMx file (`.bimx`)

| Member | Use |
Expand Down
35 changes: 35 additions & 0 deletions src/PlaIO/CompressedPool.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using PlaIO.Internal;

namespace PlaIO;

/// <summary>A pool of QuickLZ frames stored between indexed FDB block payloads.</summary>
public sealed class CompressedPool
{
private readonly PlaDocument _document;

internal CompressedPool(PlaDocument document, PoolRange range)
{
_document = document;
Offset = range.Offset;
End = range.End;
TotalDecompressed = range.UnpackedSize;
BlockCount = range.FrameCount;
}

/// <summary>The file offset of the pool marker.</summary>
public long Offset { get; }

/// <summary>The first file offset after the pool.</summary>
public long End { get; }

/// <summary>The declared size of the decoded pool in bytes.</summary>
public long TotalDecompressed { get; }

/// <summary>The number of QuickLZ frames in the pool.</summary>
public int BlockCount { get; }

/// <summary>Decodes the frames and concatenates their payloads in file order.</summary>
/// <exception cref="InvalidDataException">The pool is malformed or a frame fails to decode.</exception>
/// <exception cref="ObjectDisposedException">The owning document is disposed.</exception>
public byte[] Decode() => _document.DecodePool(Offset);
}
149 changes: 149 additions & 0 deletions src/PlaIO/Internal/PoolReader.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
using System.Buffers.Binary;

namespace PlaIO.Internal;

internal readonly record struct PoolRange(long Offset, long End, int UnpackedSize, int FrameCount);

internal static class PoolReader
{
private static ReadOnlySpan<byte> Signature =>
[0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];

private const int PoolHeaderLength = 32;
private const int MinFrameEntryLength = 7;

private readonly record struct Frame(long PayloadOffset, int PayloadLength, int UnpackedSize, bool IsCompressed);

public static List<PoolRange> Find(Stream stream, long length, IReadOnlyList<PlaBlock> blocks)
{
var pools = new List<PoolRange>();
var buffer = new byte[64 * 1024];
long pos = FdbFormat.FirstBlockOffset;
foreach (var block in blocks)
{
pos = ScanRange(stream, length, pos, block.PayloadOffset, buffer, pools);
pos = Math.Max(pos, block.End);
}
ScanRange(stream, length, pos, length, buffer, pools);
return pools;
}

private static long ScanRange(Stream stream, long length, long pos, long end, byte[] buffer, List<PoolRange> pools)
{
while (end - pos >= Signature.Length)
{
long windowStart = pos;
var window = buffer.AsSpan(0, (int)Math.Min(buffer.Length, end - windowStart));
StreamReads.ReadExactlyAt(stream, windowStart, window, "pool markers");
pos = windowStart + window.Length - Signature.Length + 1;
int from = 0;
while (window.Length - from >= Signature.Length)
{
int index = window[from..].IndexOf(Signature);
if (index < 0) break;
long offset = windowStart + from + index;
try
{
var (range, _) = ReadLayout(stream, length, offset);
pools.Add(range);
if (range.End >= windowStart + window.Length)
{
pos = range.End;
break;
}
from = (int)(range.End - windowStart);
pos = Math.Max(pos, range.End);
}
catch (InvalidDataException)
{
from += index + 1;
}
}
}
return pos;
}

public static byte[] Decode(Stream stream, long length, long offset)
{
var (range, frames) = ReadLayout(stream, length, offset);
byte[] decoded = new byte[range.UnpackedSize];
int written = 0;
foreach (var frame in frames)
{
Span<byte> output = decoded.AsSpan(written, frame.UnpackedSize);
if (frame.IsCompressed)
{
byte[] input = new byte[frame.PayloadLength];
StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, input, "a compressed pool frame");
QuickLzLevelOne.Decode(input, output);
}
else
{
StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, output, "a stored pool frame");
}
written += output.Length;
}
return decoded;
}

private static (PoolRange Range, List<Frame> Frames) ReadLayout(Stream stream, long length, long offset)
{
if (length - offset < PoolHeaderLength)
throw InvalidPool(offset, "the pool header is cut short");

Span<byte> header = stackalloc byte[PoolHeaderLength];
StreamReads.ReadExactlyAt(stream, offset, header, "a pool header");
if (!header[..Signature.Length].SequenceEqual(Signature))
throw InvalidPool(offset, "the pool marker is missing");

ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header[16..]);
ulong count = BinaryPrimitives.ReadUInt64LittleEndian(header[24..]);
long pos = offset + PoolHeaderLength;
if (total > (ulong)Array.MaxLength || count > int.MaxValue || count > (ulong)((length - pos) / MinFrameEntryLength))
throw InvalidPool(offset, "the declared pool size or frame count is outside the supported bounds");

var frames = new List<Frame>();
ulong written = 0;
Span<byte> entry = stackalloc byte[13];
for (int index = 0; index < (int)count; index++)
{
if (length - pos < MinFrameEntryLength)
throw InvalidPool(offset, $"frame {index} has a truncated header");

StreamReads.ReadExactlyAt(stream, pos, entry[..MinFrameEntryLength], "a pool frame header");
uint packed = BinaryPrimitives.ReadUInt32LittleEndian(entry);
byte flags = entry[4];
if ((flags & 0xFC) != 0x44)
throw InvalidPool(offset, $"frame {index} does not use standalone QuickLZ level 1");

bool longHeader = (flags & 2) != 0;
int headerBytes = longHeader ? 9 : 3;
if (packed < headerBytes || packed > length - pos - 4 || packed > Array.MaxLength)
throw InvalidPool(offset, $"frame {index} has an invalid packed size");

if (longHeader)
StreamReads.ReadExactlyAt(stream, pos + MinFrameEntryLength, entry[MinFrameEntryLength..], "a long pool frame header");
uint innerSize = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[5..]) : entry[5];
uint unpacked = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[9..]) : entry[6];
if (innerSize != packed || unpacked > total - written)
throw InvalidPool(offset, $"frame {index} declares inconsistent sizes");

bool compressed = (flags & 1) != 0;
int payloadLength = (int)packed - headerBytes;
// The longest match emits 255 bytes for a three-byte reference.
if (compressed ? unpacked > 85L * payloadLength : unpacked != payloadLength)
throw InvalidPool(offset, $"frame {index} cannot produce its declared size");

frames.Add(new Frame(pos + 4 + headerBytes, payloadLength, (int)unpacked, compressed));
written += unpacked;
pos += 4 + (long)packed;
}

if (written != total)
throw InvalidPool(offset, "the frame sizes do not add up to the declared pool size");
return (new PoolRange(offset, pos, (int)total, frames.Count), frames);
}

private static InvalidDataException InvalidPool(long offset, string reason) =>
new($"Pool at 0x{offset:x}: {reason}.");
}
77 changes: 77 additions & 0 deletions src/PlaIO/Internal/QuickLzLevelOne.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
using System.Buffers.Binary;

namespace PlaIO.Internal;

// QuickLZ 1.5.0 level 1, without streaming. See docs/public-api.md for the public format reference.
internal static class QuickLzLevelOne
{
public static void Decode(ReadOnlySpan<byte> input, Span<byte> output)
{
Span<int> matches = stackalloc int[4096];
matches.Fill(-1);
uint control = 1;
int written = 0;
int nextHash = 0;
bool tail = false;

while (written < output.Length)
{
if (control == 1)
{
control = TakeWord(ref input, sizeof(uint));
if ((control & 0x80000000) == 0)
throw new InvalidDataException("A QuickLZ control word has no sentinel bit.");
}

if (!tail && (control & 1) != 0)
{
uint token = TakeWord(ref input, sizeof(ushort));
int lengthCode = (int)(token & 15);
int count = lengthCode == 0 ? (int)TakeWord(ref input, sizeof(byte)) : lengthCode + 2;
int start = matches[(int)(token >> 4)];
if (start < 0 || count < 3 || count > output.Length - written)
throw new InvalidDataException("A QuickLZ reference has an invalid position or length.");

int matchStart = written;
for (int i = 0; i < count; i++)
output[written++] = output[start + i];
Register(matches, output, nextHash, matchStart + 1);
nextHash = written;
}
else
{
tail |= output.Length - written <= 10;
output[written++] = (byte)TakeWord(ref input, sizeof(byte));
if (!tail && nextHash + 3 <= written)
{
Register(matches, output, nextHash, written - 2);
nextHash = written - 2;
}
}
control >>= 1;
}
}

private static void Register(Span<int> matches, ReadOnlySpan<byte> output, int start, int end)
{
for (int pos = start; pos < end; pos++)
{
int word = output[pos] | (output[pos + 1] << 8) | (output[pos + 2] << 16);
matches[(word ^ (word >> 12)) & 0xFFF] = pos;
}
}

private static uint TakeWord(ref ReadOnlySpan<byte> input, int width)
{
if (width > input.Length)
throw new InvalidDataException("A QuickLZ frame ends inside a token.");
uint word = width switch
{
sizeof(uint) => BinaryPrimitives.ReadUInt32LittleEndian(input),
sizeof(ushort) => BinaryPrimitives.ReadUInt16LittleEndian(input),
_ => input[0],
};
input = input[width..];
return word;
}
}
13 changes: 13 additions & 0 deletions src/PlaIO/PlaDocument.cs
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,19 @@ public IReadOnlyList<ObjectDirectory> FindDirectories()
return found.AsReadOnly();
}

/// <summary>Finds compressed pools whose marker and frame headers parse, outside indexed block payloads.</summary>
public IReadOnlyList<CompressedPool> FindCompressedPools()
{
ObjectDisposedException.ThrowIf(_disposed, this);
return PoolReader.Find(_stream, Length, Blocks).Select(range => new CompressedPool(this, range)).ToList().AsReadOnly();
}

internal byte[] DecodePool(long offset)
{
ObjectDisposedException.ThrowIf(_disposed, this);
return PoolReader.Decode(_stream, Length, offset);
}

/// <summary>Releases the stream unless the document was opened with leaveOpen.</summary>
public void Dispose()
{
Expand Down
Loading
Loading