Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ API between minor releases.
- `PlaDocument.FindCompressedPools()` finds pools outside indexed block payloads. `CompressedPool.Decode()` reads
stored and QuickLZ level 1 frames using the public format description and saved-file verification.

- `CompressedPool.ReadSubgroups()` reads BL-tagged records from decoded pools and reports incomplete payloads.

### Changed

- The release workflow follows the IOExtend release order: the NuGet packages are pushed before the GitHub release is
Expand Down
13 changes: 13 additions & 0 deletions docs/public-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,19 @@ independent decoding of saved-file frames verified the correspondence. PlaIO has
| `CompressedPool.TotalDecompressed` and `BlockCount` | Decoded byte count and number of frames in the pool. |
| `CompressedPool.Decode()` | Decoded frame payloads in order. Throws `InvalidDataException` on invalid framing or data and `ObjectDisposedException` after disposing the document. |

### Pool subgroups

The subgroup header uses the FDB size field widths and may carry a GUID. It does not contain the file-level extra field.
Offsets are relative to the decoded pool. The scanner retries an unmatched BL candidate at the next byte.

| Member | Use |
|---|---|
| `CompressedPool.ReadSubgroups()` | Decodes the pool and returns the recognised BL-tagged records in pool order. |
| `PoolSubgroup.Offset` | Start of the subgroup header in the decoded pool. |
| `PoolSubgroup.Kind`, `Version`, `Flags` and `ObjectId` | Raw header metadata and the optional GUID. |
| `PoolSubgroup.Mode` and `Data` | Raw or zero-mask payload bytes. Mode 2 is reported with empty data. |
| `PoolSubgroup.Partial` | True for unavailable mode 2 data or an incomplete zero-mask literal. A zero-mask stream ending at a group boundary can leave valid zero padding. |

## BIMx file (`.bimx`)

| Member | Use |
Expand Down
5 changes: 5 additions & 0 deletions src/PlaIO/CompressedPool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,9 @@ internal CompressedPool(PlaDocument document, PoolRange range)
/// <exception cref="InvalidDataException">The pool is malformed or a frame fails to decode.</exception>
/// <exception cref="ObjectDisposedException">The owning document is disposed.</exception>
public byte[] Decode() => _document.DecodePool(Offset);

/// <summary>Decodes the pool and reads BL-tagged records in pool order.</summary>
/// <exception cref="InvalidDataException">The pool does not decode.</exception>
/// <exception cref="ObjectDisposedException">The owning document is disposed.</exception>
public IReadOnlyList<PoolSubgroup> ReadSubgroups() => PoolSubgroupReader.Read(Decode());
}
7 changes: 4 additions & 3 deletions src/PlaIO/Internal/BlockHeaderParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ internal static class BlockHeaderParser
/// <summary>Parses a block header that starts at <paramref name="at"/>[0], found at file offset <paramref name="offset"/>.
/// The span may end before a full header near the end of the file. Returns false at the first rule that fails; never
/// throws.</summary>
public static bool TryParse(ReadOnlySpan<byte> at, long offset, long fileLength, [NotNullWhen(true)] out PlaBlock? block)
public static bool TryParse(ReadOnlySpan<byte> at, long offset, long fileLength, [NotNullWhen(true)] out PlaBlock? block, bool poolSubgroup = false)
{
block = null;
if (at.Length < 11) return false;
Expand Down Expand Up @@ -39,7 +39,7 @@ public static bool TryParse(ReadOnlySpan<byte> at, long offset, long fileLength,
};
p += width;
if (packed > long.MaxValue) return false;
if (SizeClass.Of(packed) != sizeClass) return false;
if (!poolSubgroup && SizeClass.Of(packed) != sizeClass) return false;

var mode = (flags & FdbFormat.FlagZeroMask) != 0 ? PlaBlockMode.ZeroMask
: (flags & FdbFormat.FlagLz) != 0 ? PlaBlockMode.Lz
Expand All @@ -53,7 +53,8 @@ public static bool TryParse(ReadOnlySpan<byte> at, long offset, long fileLength,
}

uint? extra = null;
if ((flags & FdbFormat.FlagExtra) != 0)
// Pool subgroups use the file header's size widths but have no separate extra field.
if (!poolSubgroup && (flags & FdbFormat.FlagExtra) != 0)
{
if (at.Length < p + 4) return false;
extra = BinaryPrimitives.ReadUInt32LittleEndian(at[p..]);
Expand Down
20 changes: 17 additions & 3 deletions src/PlaIO/Internal/PayloadDecoder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,20 @@ internal static class PayloadDecoder
/// left after that are allowed.</summary>
public static int DecodeZeroMask(ReadOnlySpan<byte> src, Span<byte> dst)
{
int consumed = 0, written = 0;
int consumed = DecodeZeroMask(src, dst, out int written, out _);
if (written != dst.Length) throw Short(consumed, src.Length, written, dst.Length);
return consumed;
}

/// <summary>Decodes the available data and reports whether input ended during a literal group.</summary>
public static int DecodeZeroMask(ReadOnlySpan<byte> src, Span<byte> dst, out int written, out bool truncatedLiteral)
{
int consumed = 0;
written = 0;
truncatedLiteral = false;
while (written < dst.Length)
{
if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length);
if (consumed >= src.Length) break;
int mask = src[consumed++];
for (int bit = 0; bit < 8 && written < dst.Length; bit++)
{
Expand All @@ -20,7 +30,11 @@ public static int DecodeZeroMask(ReadOnlySpan<byte> src, Span<byte> dst)
dst[written++] = 0;
continue;
}
if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length);
if (consumed >= src.Length)
{
truncatedLiteral = true;
return consumed;
}
dst[written++] = src[consumed++];
}
}
Expand Down
43 changes: 43 additions & 0 deletions src/PlaIO/Internal/PoolSubgroupReader.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
namespace PlaIO.Internal;

internal static class PoolSubgroupReader
{
public static IReadOnlyList<PoolSubgroup> Read(byte[] pool)
{
var subgroups = new List<PoolSubgroup>();
int pos = 0;
while (pool.Length - pos >= FdbFormat.MinBlockHeaderLength)
{
int next = pool.AsSpan(pos).IndexOf("BL"u8);
if (next < 0) break;
pos += next;
if (!BlockHeaderParser.TryParse(pool.AsSpan(pos), pos, pool.Length, out var header, poolSubgroup: true))
{
pos++;
continue;
}

ReadOnlySpan<byte> payload = pool.AsSpan((int)header.PayloadOffset, (int)header.PackedSize);
byte[] data;
bool partial;
switch (header.Mode)
{
case PlaBlockMode.Raw:
data = payload.ToArray();
partial = false;
break;
case PlaBlockMode.ZeroMask:
data = new byte[header.UnpackedSize];
PayloadDecoder.DecodeZeroMask(payload, data, out _, out partial);
break;
default:
data = [];
partial = true;
break;
}
subgroups.Add(new PoolSubgroup(header, data, partial));
pos = (int)header.End;
}
return subgroups.AsReadOnly();
}
}
42 changes: 42 additions & 0 deletions src/PlaIO/PoolSubgroup.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
namespace PlaIO;

/// <summary>A BL-tagged record in a decoded compressed pool.</summary>
public sealed class PoolSubgroup
{
internal PoolSubgroup(PlaBlock header, byte[] data, bool partial)
{
Offset = header.Offset;
Kind = header.Kind;
Version = header.Version;
Flags = header.Flags;
Mode = header.Mode;
ObjectId = header.Guid;
Data = data;
Partial = partial;
}

/// <summary>The offset of the header in the decoded pool.</summary>
public long Offset { get; }

/// <summary>The raw class number from the header.</summary>
public int Kind { get; }

/// <summary>The version byte.</summary>
public int Version { get; }

/// <summary>The raw header flags.</summary>
public int Flags { get; }

/// <summary>The storage mode of the payload.</summary>
public PlaBlockMode Mode { get; }

/// <summary>The object GUID when present in the header.</summary>
public Guid? ObjectId { get; }

/// <summary>The decoded payload. Mode 2 returns empty data pending a supported decoder.</summary>
public byte[] Data { get; }

/// <summary>True when decoding is unavailable or the zero-mask stream ends inside a literal group.
/// A group-boundary ending can have a valid zero-filled tail with this property false.</summary>
public bool Partial { get; }
}
65 changes: 13 additions & 52 deletions tests/PlaIO.Tests/CompressedPoolTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,13 @@ namespace PlaIO.Tests;

public class CompressedPoolTests
{
private static ReadOnlySpan<byte> PoolMarker =>
[0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];

[Fact]
public void ReadsShortAndLongStoredFrames()
{
byte[] first = "ABC"u8.ToArray();
byte[] second = Enumerable.Repeat((byte)'Z', 220).ToArray();
byte[] bytes = Synthetic.Fdb(Pool(((uint)first.Length, Frame(first, first.Length, flags: 0x44)),
((uint)second.Length, Frame(second, second.Length, flags: 0x46))));
byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool(((uint)first.Length, Synthetic.QuickLzFrame(first, first.Length, flags: 0x44)),
((uint)second.Length, Synthetic.QuickLzFrame(second, second.Length, flags: 0x46))));
using var doc = PlaDocument.Open(new MemoryStream(bytes));

var pool = Assert.Single(doc.FindCompressedPools());
Expand All @@ -35,7 +32,7 @@ public void DecodesAnOverlappingQuickLzMatchAndLiteralTail()
byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], "ABCD"u8.ToArray(), tokenBytes, "0123456789"u8.ToArray());
byte[] expected = "ABCDABCDABCDABCD0123456789"u8.ToArray();
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(
Pool(((uint)expected.Length, Frame(body, expected.Length, flags: 0x45))))));
Synthetic.QuickLzPool(((uint)expected.Length, Synthetic.QuickLzFrame(body, expected.Length, flags: 0x45))))));

Assert.Equal(expected, Assert.Single(doc.FindCompressedPools()).Decode());
}
Expand All @@ -44,7 +41,7 @@ public void DecodesAnOverlappingQuickLzMatchAndLiteralTail()
public void DecodedPoolCanBeLargerThanTheContainer()
{
byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], new byte[4], [0, 0, 190], new byte[11]);
byte[] bytes = Synthetic.Fdb(Pool((205, Frame(body, 205, flags: 0x47))));
byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool((205, Synthetic.QuickLzFrame(body, 205, flags: 0x47))));
using var doc = PlaDocument.Open(new MemoryStream(bytes));

Assert.True(bytes.Length < 205);
Expand All @@ -54,7 +51,7 @@ public void DecodedPoolCanBeLargerThanTheContainer()
[Fact]
public void MarkerInsideAnIndexedPayloadIsIgnored()
{
byte[] pool = Pool((3, Frame([1, 2, 3], 3, flags: 0x44)));
byte[] pool = Synthetic.QuickLzPool((3, Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44)));
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(
Synthetic.Block(PlaBlockMode.Raw, pool),
pool)));
Expand All @@ -66,7 +63,7 @@ public void MarkerInsideAnIndexedPayloadIsIgnored()
[Fact]
public void FindsMarkerAcrossTheScanWindowBoundary()
{
byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Pool());
byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Synthetic.QuickLzPool());
using var doc = PlaDocument.Open(new MemoryStream(bytes));
var pool = Assert.Single(doc.FindCompressedPools());
Assert.Equal(bytes.Length, pool.End);
Expand All @@ -76,17 +73,17 @@ public void FindsMarkerAcrossTheScanWindowBoundary()
[Fact]
public void RejectsATruncatedFrameAtEndOfFile()
{
byte[] frame = Frame([1, 2, 3], 3, flags: 0x44);
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))[..^1])));
byte[] frame = Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44);
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool((3, frame))[..^1])));

Assert.Empty(doc.FindCompressedPools());
}

[Fact]
public void APoolWithInconsistentLengthsDoesNotHideAValidPool()
{
byte[] frame = Frame([1, 2, 3], 3, flags: 0x44);
byte[] bytes = Synthetic.Fdb(Pool((4, frame)), Pool((3, frame)));
byte[] frame = Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44);
byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool((4, frame)), Synthetic.QuickLzPool((3, frame)));
using var doc = PlaDocument.Open(new MemoryStream(bytes));

var pool = Assert.Single(doc.FindCompressedPools());
Expand All @@ -97,16 +94,16 @@ public void APoolWithInconsistentLengthsDoesNotHideAValidPool()
[Fact]
public void DecodeRejectsAnUninitializedMatch()
{
byte[] frame = Frame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45);
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame)))));
byte[] frame = Synthetic.QuickLzFrame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45);
using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool((3, frame)))));
var pool = Assert.Single(doc.FindCompressedPools());
Assert.Throws<InvalidDataException>(pool.Decode);
}

[Fact]
public void DecodeAfterDisposeThrows()
{
var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool())));
var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool())));
var pool = Assert.Single(doc.FindCompressedPools());
doc.Dispose();

Expand All @@ -130,40 +127,4 @@ public void SavedFilePoolFramesDecodeToTheirDeclaredSize()
if (decoded == 0) throw TestFiles.Missing("FDB fixture with QuickLZ pools");
}

private static byte[] Pool(params (uint Size, byte[] Frame)[] frames)
{
byte[] header = new byte[32];
PoolMarker.CopyTo(header);
BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), (ulong)frames.Sum(f => (long)f.Size));
BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(24), (ulong)frames.Length);
using var result = new MemoryStream();
result.Write(header);
Span<byte> size = stackalloc byte[4];
foreach (var (_, frame) in frames)
{
BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)frame.Length);
result.Write(size);
result.Write(frame);
}
return result.ToArray();
}

private static byte[] Frame(byte[] body, int size, byte flags)
{
int header = (flags & 2) == 0 ? 3 : 9;
byte[] frame = new byte[header + body.Length];
frame[0] = flags;
if (header == 3)
{
frame[1] = checked((byte)frame.Length);
frame[2] = checked((byte)size);
}
else
{
BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(1), (uint)frame.Length);
BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(5), (uint)size);
}
body.CopyTo(frame, header);
return frame;
}
}
Loading
Loading