diff --git a/CHANGELOG.md b/CHANGELOG.md
index dff1a25..91d5093 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,29 @@ All notable changes are documented here. The format follows [Keep a Changelog](h
and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html); versions before 1.0 may change the
API between minor releases.
+## [Unreleased]
+
+### Added
+
+- `CompressedPool.ReadSubgroups` and `PoolSubgroup`: the object records the object database keeps inside a pool —
+ BL-tagged subgroups with the object's class id (`Kind`), `Version`, `Flags`, `ObjectId` (a GUID when the header
+ carries one), the payload's storage (`Mode`: raw or zero mask) and the decoded payload (`Data`). The headers follow
+ the file's block layout (a u16 packed size for size classes 0..6, a u32 for 7..14); a zero-mask stream that ends
+ before its declared unpacked size leaves the tail of the buffer zero. Mode 2 (LZ) subgroups have not been met inside
+ the pools examined so far and are reported with an empty payload. The records are found by scanning for the BL tag
+ and accepting the ones whose header parses; a rejected candidate is retried one byte on. Observed payloads store text
+ as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers.
+
+- `PlaDocument.FindCompressedPools` and `CompressedPool`: the compressed storage pools of the object database that the
+ file holds in the gaps between blocks, each starting with a 16-byte compression sign, a declared decompressed total
+ and its blocks (each memcpy or LZ77-triple coded, at most one megabyte and change of packed payload per block). The
+ object database keeps its groups, journals and library data in these pools; a pool is not a block and does not take
+ part in the block index. `FindCompressedPools` finds every pool whose header and block headers parse and whose
+ declared sizes tile the total (computed on each call; the scan reads the file in windows and skips the indexed block
+ payloads, so a sign inside them is never returned as a pool); `Decode` produces the decoded bytes on request. The
+ layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample
+ files.
+
## [0.1.0] - 2026-11-12
### Added
diff --git a/docs/public-api.md b/docs/public-api.md
index f3e6b94..aadea5b 100644
--- a/docs/public-api.md
+++ b/docs/public-api.md
@@ -1,8 +1,9 @@
# Public API
The `PlaIO` package is the supported surface listed here; every public type is in the namespace `PlaIO`. Everything else
-(the block scanner, the header parser, the payload decoder, the directory search, the PNG chunk walker) is `internal` in
-`PlaIO.Internal`; it can change in any release and is visible only to the test project through `InternalsVisibleTo`.
+(the block scanner, the header parser, the payload decoder, the directory search, the pool decoder, the subgroup parser,
+the PNG chunk walker) is `internal` in `PlaIO.Internal`; it can change in any release and is visible only to the test
+project through `InternalsVisibleTo`.
## File kind
@@ -30,6 +31,33 @@ The `PlaIO` package is the supported surface listed here; every public type is i
| `ObjectDirectory` | A run of directory records inside a raw block: `Block`, `RecordOffset` (payload offset of the first record), `Phase` (`RecordOffset % 30`), `Records`, `AbuttingPairs`, `OverlappingPairs`, `DuplicatePositions`. The run is an estimate and can be shorter than the real directory: it stops after more than 8 consecutive records without an abutting neighbour. |
| `ObjectDirectoryRecord` | The 30-byte record `[u16 class][u64 pos][u32 count][GUID16]` as `ClassNumber`, `Position`, `Count`, `ObjectId`. The units of position and count are not documented, and the class number is not mapped to a class name. |
+## Compressed pools (the object database)
+
+The object database keeps its groups, journals and library data in compressed pools that sit in the gaps between blocks.
+A pool starts with a 16-byte compression sign, then the declared decompressed total and the block count (both unsigned
+64-bit little endian), then that many blocks, each a u32 compressed size followed by the block: a flag byte (bit 0: the
+payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise single bytes), the packed size
+(equal to the compressed size), the unpacked size and the payload. The blocks decode to exactly the declared total. The
+layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample files.
+
+| Member | Use |
+|---|---|
+| `PlaDocument.FindCompressedPools()` | Every pool whose header and block headers parse and whose declared sizes tile the total, in file order. Recomputed on each call; the scan reads the file in windows and skips the indexed block payloads, so a sign inside them is never returned as a pool. |
+| `CompressedPool.Offset` | File offset of the pool's compression sign. |
+| `CompressedPool.End` | File offset past the last block of the pool. |
+| `CompressedPool.TotalDecompressed` | The declared size of the decoded pool in bytes. |
+| `CompressedPool.BlockCount` | The number of blocks; at most one megabyte and change of packed payload per block. |
+| `CompressedPool.Decode()` | The decoded bytes: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache. Throws `InvalidDataException` when a block does not decode or the blocks do not add up to the total, and `ObjectDisposedException` when the document is disposed. |
+| `CompressedPool.ReadSubgroups()` | The object records inside the pool: BL-tagged subgroups whose headers parse, with their payloads decoded. Recomputed on each call. A rejected BL candidate is retried one byte on. |
+| `PoolSubgroup.Offset` | Offset of the subgroup header inside the decoded pool. |
+| `PoolSubgroup.Kind` | The object's class id (the kind field of the BL header). |
+| `PoolSubgroup.Version` | The header's version byte, 0 or 1. |
+| `PoolSubgroup.Flags` | The header's flags: the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the payload modes, the low 4 bits are the packed size class. |
+| `PoolSubgroup.Mode` | The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but not decoded; it has not been met inside the pools examined so far. |
+| `PoolSubgroup.ObjectId` | The object's GUID, present when flag 0x20 is set. |
+| `PoolSubgroup.Partial` | True when decoding stopped at a truncated literal or the payload could not be decoded (unsupported mode or size). A zero-mask stream ending at a group boundary has valid zero padding. |
+| `PoolSubgroup.Data` | The decoded payload; empty for mode 2 subgroups. A zero-mask stream that ends before its declared unpacked size leaves the tail of the buffer zero. Observed payloads store text as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. |
+
## BIMx file (`.bimx`)
| Member | Use |
@@ -46,7 +74,8 @@ The `PlaIO` package is the supported surface listed here; every public type is i
`PlaDocument` and `BimxDocument` never load the whole file into memory: the block index is built by scanning through the
stream once, at `Open`, and payloads and the cover are read on request. A document is not thread-safe; use one per thread.
-After `Dispose`, `ReadPayload`, `FindDirectories` and `ReadCover` throw `ObjectDisposedException`.
+After `Dispose`, `ReadPayload`, `FindDirectories`, `FindCompressedPools`, `CompressedPool.Decode`,
+`CompressedPool.ReadSubgroups` and `ReadCover` throw `ObjectDisposedException`.
## Diagnostics
diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs
new file mode 100644
index 0000000..ccc3771
--- /dev/null
+++ b/src/PlaIO/CompressedPool.cs
@@ -0,0 +1,47 @@
+namespace PlaIO;
+
+/// A compressed storage pool of the ODB object database that the file holds in a gap between blocks: the
+/// compression sign, the declared decompressed total and the blocks (each memcpy or LZ77-triple coded). The object
+/// database keeps its groups, journals and library data in such pools; a pool is not a block and does not take part in
+/// the block index or the payloads. The decoded bytes are produced on request by ; the cost of a
+/// call grows with the pool size. The layout and the codec follow the format notes of this repository and are confirmed
+/// against the sample files.
+public sealed class CompressedPool
+{
+ private readonly PlaDocument _document;
+
+ internal CompressedPool(PlaDocument document, long offset, long end, long total, int blockCount)
+ {
+ _document = document;
+ Offset = offset;
+ End = end;
+ TotalDecompressed = total;
+ BlockCount = blockCount;
+ }
+
+ /// File offset of the pool's compression sign.
+ public long Offset { get; }
+
+ /// File offset past the last block of the pool.
+ public long End { get; }
+
+ /// The declared size of the decoded pool in bytes; the blocks decode to exactly this size.
+ public long TotalDecompressed { get; }
+
+ /// The number of blocks; at most one megabyte and change of packed payload per block.
+ public int BlockCount { get; }
+
+ /// Decodes the pool: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache; each
+ /// call decodes again.
+ /// A block does not decode or the blocks do not add up to
+ /// .
+ /// The document is disposed.
+ public byte[] Decode() => _document.DecodePool(Offset);
+
+ /// Decodes the pool and reads its object records: the BL-tagged subgroups whose headers parse, with their
+ /// payloads decoded (raw or zero mask; mode 2 is reported with an empty payload). Recomputed on each call; a
+ /// rejected BL candidate is retried one byte on, so a short false header can skip over a real record.
+ /// The pool does not decode.
+ /// The document is disposed.
+ public IReadOnlyList ReadSubgroups() => Internal.SubgroupParser.Parse(Decode());
+}
diff --git a/src/PlaIO/Internal/PayloadDecoder.cs b/src/PlaIO/Internal/PayloadDecoder.cs
index 4d9e88d..e9bc841 100644
--- a/src/PlaIO/Internal/PayloadDecoder.cs
+++ b/src/PlaIO/Internal/PayloadDecoder.cs
@@ -2,16 +2,53 @@ namespace PlaIO.Internal;
internal static class PayloadDecoder
{
+ /// How a zero-mask stream ended relative to the destination buffer.
+ public enum ZeroMaskEnd
+ {
+ /// The destination buffer is full.
+ Complete,
+ /// The stream ended exactly at a group boundary before the buffer is full; the tail of the buffer stays
+ /// zero. Valid for the pool subgroups, whose declared size is the buffer size.
+ StreamEnd,
+ /// The stream ended inside a group: a mask byte was read whose set bit takes a stream byte that is not
+ /// there. A truncated literal, never a valid padding.
+ MidGroup,
+ }
+
/// Mode 1: a stream of groups, each a mask byte followed by one byte per set bit. Bits are read from the least
/// significant; a set bit takes the next stream byte, a clear bit writes 0x00. Decoding ends when
- /// is full, so bits of the last mask past the end consume nothing. Returns the number of source bytes consumed; bytes
- /// left after that are allowed.
+ /// is full, so bits of the last mask past the end consume nothing; the status tells how the
+ /// stream ended when is not full. Returns the number of source bytes consumed; bytes left after
+ /// that are allowed.
+ public static int DecodeZeroMask(ReadOnlySpan src, Span dst, out ZeroMaskEnd end)
+ {
+ return DecodeCore(src, dst, out end, out _);
+ }
+
+ /// Mode 1 for the file blocks: the stream must fill the buffer exactly; a stream that ends before that — at a
+ /// group boundary or inside a group — does not decode. Returns the number of source bytes consumed; bytes left after
+ /// that are allowed.
+ /// The stream ends before the buffer is full.
public static int DecodeZeroMask(ReadOnlySpan src, Span dst)
{
- int consumed = 0, written = 0;
+ int consumed = DecodeCore(src, dst, out _, out var written);
+ if (written < dst.Length)
+ throw Short(consumed, src.Length, written, dst.Length);
+ return consumed;
+ }
+
+ private static int DecodeCore(ReadOnlySpan src, Span dst, out ZeroMaskEnd end, out int written)
+ {
+ int consumed = 0;
+ written = 0;
+ end = ZeroMaskEnd.Complete;
while (written < dst.Length)
{
- if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length);
+ if (consumed >= src.Length)
+ {
+ end = ZeroMaskEnd.StreamEnd;
+ return consumed;
+ }
int mask = src[consumed++];
for (int bit = 0; bit < 8 && written < dst.Length; bit++)
{
@@ -20,7 +57,11 @@ public static int DecodeZeroMask(ReadOnlySpan src, Span dst)
dst[written++] = 0;
continue;
}
- if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length);
+ if (consumed >= src.Length)
+ {
+ end = ZeroMaskEnd.MidGroup;
+ return consumed;
+ }
dst[written++] = src[consumed++];
}
}
diff --git a/src/PlaIO/Internal/PoolDecoder.cs b/src/PlaIO/Internal/PoolDecoder.cs
new file mode 100644
index 0000000..35de0db
--- /dev/null
+++ b/src/PlaIO/Internal/PoolDecoder.cs
@@ -0,0 +1,294 @@
+using System.Buffers.Binary;
+
+namespace PlaIO.Internal;
+
+/// The compressed storage pools that live in the gaps between blocks. A pool starts with the 16-byte compression
+/// sign, then the decompressed total and the block count (both unsigned 64-bit little endian), then that many blocks, each
+/// a u32 compressed size followed by the block. A block is a flag byte (bit 0: the payload is LZ77-triple coded, otherwise
+/// copied; bit 1: the sizes are 32-bit, otherwise bytes), the packed size (equal to the block's compressed size), the
+/// unpacked size and the payload. The blocks decode to exactly the declared total. The layout and the codec follow the
+/// format notes of this repository and are confirmed against the sample files.
+internal static class PoolDecoder
+{
+ /// The compression sign that opens every compressed pool.
+ public static ReadOnlySpan Sign => [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];
+
+ /// At most one megabyte and change of packed payload per block; the reader allocates a buffer of this size
+ /// plus a byte for the block.
+ public const int MaxCompressedBlock = 1048976;
+
+ /// The pools of the stream: every compression sign whose header parses and whose block headers tile the
+ /// declared total inside the file. The scan continues past an accepted pool; a rejected sign is retried one byte on.
+ /// The ranges are skipped entirely: a sign inside them is not considered (the document
+ /// passes the byte ranges of its indexed blocks, whose payloads can carry the sign as data). Returns the pool
+ /// descriptions in file order.
+ public static List<(long Offset, long End, ulong Total, int BlockCount)> Find(Stream stream, long length, IReadOnlyList<(long Start, long End)>? excluded)
+ {
+ var pools = new List<(long Offset, long End, ulong Total, int BlockCount)>();
+ var window = new byte[1 << 20];
+ long pos = 0;
+ while (pos + Sign.Length <= length)
+ {
+ // Step over an excluded range whole: its signs are data, not pool starts.
+ if (excluded is not null)
+ foreach (var (start, end) in excluded)
+ {
+ if (pos >= start && pos < end) pos = end;
+ }
+ if (pos + Sign.Length > length) break;
+ int count = (int)Math.Min(window.Length, length - pos);
+ StreamReads.ReadExactlyAt(stream, pos, window.AsSpan(0, count), "the pool scan");
+ int at = IndexOfSign(window.AsSpan(0, count));
+ if (at < 0 || pos + at + Sign.Length > length)
+ {
+ // The next scan start keeps a full sign inside the next window; a sign can straddle the boundary.
+ pos = count == window.Length ? pos + count - (Sign.Length - 1) : length;
+ continue;
+ }
+ long signAt = pos + at;
+ if (Inside(excluded, signAt))
+ {
+ pos = signAt + 1;
+ continue;
+ }
+ if (TryValidate(stream, length, signAt, out var pool))
+ {
+ pools.Add(pool);
+ pos = pool.End;
+ }
+ else
+ {
+ pos = signAt + 1;
+ }
+ }
+ return pools;
+ }
+
+ /// Decodes the pool at : its blocks, each memcpy or LZ77-triple, concatenated.
+ /// Throws when the blocks do not decode or do not add up to the declared total.
+ public static byte[] Decode(Stream stream, long offset, long length)
+ {
+ var (total, blocks) = ReadBlocks(stream, offset, length);
+ if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit");
+ var output = new byte[total];
+ long written = 0;
+ foreach (var (payloadAt, packed, unpacked) in blocks)
+ {
+ var payload = new byte[packed];
+ StreamReads.ReadExactlyAt(stream, payloadAt, payload, $"pool at 0x{offset:x}");
+ int flag = payload[0];
+ int header = (flag & 0b10) != 0 ? 9 : 3;
+ int produced = (flag & 1) != 0
+ ? DecodeTriple(payload, output.AsSpan((int)written, (int)unpacked), (int)unpacked)
+ : CopyPayload(payload[header..], output.AsSpan((int)written, (int)unpacked));
+ if (produced != (int)unpacked)
+ throw new InvalidDataException($"Pool at 0x{offset:x}: block at 0x{payloadAt:x} produced {produced} of {unpacked} bytes.");
+ written += unpacked;
+ }
+ if (written != (long)total)
+ throw new InvalidDataException($"Pool at 0x{offset:x}: blocks produced {written} of {total} bytes.");
+ return output;
+ }
+
+ /// The block layout of the pool whose sign sits at : the u64 total and block count
+ /// after the sign, then one entry per block with its payload offset, packed and unpacked size. The declared total can
+ /// exceed the container (the codec is compressing); only the array limit and the file bounds of every block header
+ /// and payload are checked. The sizes must tile the declared total exactly.
+ private static (ulong Total, List<(long PayloadAt, int Packed, int Unpacked)> Blocks) ReadBlocks(Stream stream, long offset, long length)
+ {
+ var header = new byte[16];
+ if (offset + Sign.Length + header.Length > length) throw Bad(offset, "its header runs past the file");
+ StreamReads.ReadExactlyAt(stream, offset + Sign.Length, header, $"pool at 0x{offset:x}");
+ ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header);
+ ulong blockCount = BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(8));
+ if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit");
+ var blocks = new List<(long, int, int)>();
+ long pos = offset + Sign.Length + header.Length;
+ ulong sum = 0;
+ for (ulong i = 0; i < blockCount; i++)
+ {
+ var sizeBytes = new byte[4];
+ if (pos + 4 > length) throw Bad(offset, "the block list runs past the file");
+ StreamReads.ReadExactlyAt(stream, pos, sizeBytes, $"pool at 0x{offset:x}");
+ uint packed = BinaryPrimitives.ReadUInt32LittleEndian(sizeBytes);
+ pos += 4;
+ if (packed < 3 || packed > MaxCompressedBlock || pos + packed > length) throw Bad(offset, $"block {i} declares {packed} packed bytes");
+ var flagBytes = new byte[1];
+ StreamReads.ReadExactlyAt(stream, pos, flagBytes, $"pool at 0x{offset:x}");
+ int flag = flagBytes[0];
+ bool big = (flag & 0b10) != 0;
+ int headerLength = big ? 9 : 3;
+ if (packed < headerLength) throw Bad(offset, $"block {i} declares {packed} packed bytes, less than its {headerLength}-byte header");
+ var head = new byte[headerLength];
+ StreamReads.ReadExactlyAt(stream, pos, head, $"pool at 0x{offset:x}");
+ uint declaredPacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(1)) : head[1];
+ if (declaredPacked != packed) throw Bad(offset, $"block {i} header declares {declaredPacked} packed bytes, the pool lists {packed}");
+ uint unpacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(5)) : head[2];
+ if (sum + unpacked > total) throw Bad(offset, $"block {i} overruns the declared total");
+ blocks.Add((pos, (int)packed, (int)unpacked));
+ sum += unpacked;
+ pos += packed;
+ }
+ if (sum != total) throw Bad(offset, $"its blocks add up to {sum}, not the declared {total}");
+ return (total, blocks);
+ }
+
+ /// Whether the header and every block header of the pool at parse and the declared
+ /// sizes tile the total inside the file.
+ private static bool TryValidate(Stream stream, long length, long offset, out (long Offset, long End, ulong Total, int BlockCount) pool)
+ {
+ pool = default;
+ try
+ {
+ var (total, blocks) = ReadBlocks(stream, offset, length);
+ pool = (offset, blocks.Count > 0 ? blocks[^1].PayloadAt + blocks[^1].Packed : offset + Sign.Length + 16, total, blocks.Count);
+ return true;
+ }
+ catch (InvalidDataException)
+ {
+ return false;
+ }
+ }
+
+ /// The first offset of the compression sign in , or -1.
+ private static int IndexOfSign(ReadOnlySpan window)
+ {
+ int at = 0;
+ while (true)
+ {
+ int next = window[at..].IndexOf(Sign[0]);
+ if (next < 0) return -1;
+ at += next;
+ if (at + Sign.Length > window.Length) return -1;
+ if (window.Slice(at, Sign.Length).SequenceEqual(Sign)) return at;
+ at++;
+ }
+ }
+
+ /// Whether falls inside one of the excluded ranges.
+ private static bool Inside(IReadOnlyList<(long Start, long End)>? excluded, long at)
+ {
+ if (excluded is null) return false;
+ foreach (var (start, end) in excluded)
+ if (start <= at && at < end) return true;
+ return false;
+ }
+
+ /// The payload of a memcpy block (the bytes after the block header, already sliced), at most
+ /// bytes.
+ private static int CopyPayload(ReadOnlySpan block, Span dst)
+ {
+ int source = Math.Min(block.Length, dst.Length);
+ block[..source].CopyTo(dst);
+ return source;
+ }
+
+ /// The LZ77-triple codec. The stream is a little-endian flag word (a 1 starts a new word) and then, per flag
+ /// bit: a set bit takes a token whose low 16 bits name a slot of the pointer table (the top 12 bits, shifted right by
+ /// 4) and a length ((the low 4 bits) + 2, or a following byte when 0); the copy reads from that slot and runs in
+ /// 3-byte steps over unsigned 32-bit reads, so the last step may write up to two bytes past the length. A clear bit
+ /// takes a token whose first four bytes go to the output, of which the literal-length table keeps the first. The last
+ /// ten bytes go out one by one. Every position up to a few bytes behind the output position is registered into the
+ /// 4096-slot table, hashed from the four output bytes at position+1. An out-of-range slot or length stops the
+ /// stream.
+ private static int DecodeTriple(ReadOnlySpan block, Span output, int unpacked)
+ {
+ var table = new long[4096];
+ Array.Fill(table, -1);
+
+ bool big = (block[0] & 0b10) != 0;
+ int packedSize = big ? (int)BinaryPrimitives.ReadUInt32LittleEndian(block[1..]) : block[1];
+ int endIn = packedSize; // the last valid input index is endIn - 1
+ int src = 3 + (big ? 6 : 0); // the payload starts after the header
+ int outPos = 0;
+ uint flags = 1;
+ long cursor = -1;
+ int outEnd = unpacked - 1;
+
+ uint token = 0;
+ while (true)
+ {
+ while (true)
+ {
+ if (flags == 1)
+ {
+ if (src + 3 > endIn - 1) return outPos;
+ flags = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src));
+ src += 4;
+ }
+ if (src + 3 > endIn - 1) return outPos;
+ token = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src));
+ if ((flags & 1) == 0) break;
+
+ flags >>= 1;
+ src += 2;
+ long source = table[(ushort)token >> 4];
+ int length;
+ int low = (int)(token & 0xF);
+ if (low != 0) length = low + 2;
+ else
+ {
+ length = block[src];
+ src += 1;
+ }
+ if (source < 0 || source > outPos - 3 || length > outEnd - outPos - 3) return outPos;
+ int step = 0;
+ while (step < length)
+ {
+ // One 32-bit read per 3-byte step; the tail may run up to two bytes past the length.
+ output[outPos + step] = output[(int)source + step];
+ output[outPos + step + 1] = output[(int)source + step + 1];
+ output[outPos + step + 2] = output[(int)source + step + 2];
+ output[outPos + step + 3] = output[(int)source + step + 3];
+ step += 3;
+ }
+ outPos += length;
+ Register(table, ref cursor, outPos - length, output);
+ cursor = outPos - 1;
+ }
+
+ if (outPos >= outEnd - 10) break;
+ output[outPos] = (byte)token;
+ output[outPos + 1] = (byte)(token >> 8);
+ output[outPos + 2] = (byte)(token >> 16);
+ output[outPos + 3] = (byte)(token >> 24);
+ int literals = LiteralLengths[flags & 0xF];
+ outPos += literals;
+ flags >>= literals;
+ src += literals;
+ Register(table, ref cursor, outPos - 3, output);
+ }
+
+ while (outPos <= outEnd)
+ {
+ uint f = flags;
+ if (flags == 1) flags = 0x80000000;
+ int at = f == 1 ? src + 4 : src;
+ if (at >= endIn) return outPos;
+ output[outPos] = block[at];
+ src = at + 1;
+ outPos++;
+ flags >>= 1;
+ }
+ Register(table, ref cursor, outEnd - 3, output);
+ return unpacked;
+ }
+
+ private static readonly int[] LiteralLengths = [4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0];
+
+ /// Registers every output position below that the cursor has not reached yet: the
+ /// hash of the four output bytes at position+1 indexes the slot that stores the position.
+ private static void Register(long[] table, ref long cursor, long limit, ReadOnlySpan output)
+ {
+ while (cursor < limit)
+ {
+ int at = (int)(cursor + 1);
+ uint word = (uint)(output[at] | output[at + 1] << 8 | output[at + 2] << 16 | output[at + 3] << 24);
+ int hash = (int)(((ushort)word ^ (ushort)(word >> 12)) & 0xFFF);
+ table[hash] = cursor + 1;
+ cursor++;
+ }
+ }
+
+ private static InvalidDataException Bad(long offset, string what) => new($"Pool at 0x{offset:x}: {what}.");
+}
diff --git a/src/PlaIO/Internal/SubgroupParser.cs b/src/PlaIO/Internal/SubgroupParser.cs
new file mode 100644
index 0000000..f9e4449
--- /dev/null
+++ b/src/PlaIO/Internal/SubgroupParser.cs
@@ -0,0 +1,96 @@
+using System.Buffers.Binary;
+
+namespace PlaIO.Internal;
+
+/// The objects the object database keeps inside a compressed pool: BL-tagged records that follow the same header
+/// layout as the file's blocks — tag, id, kind (the object's class id), version, flags (marker 0x8080, optional GUID,
+/// mode bits), the packed size (a u16 for size classes 0..6, a u32 for 7..14, a u64 for 15) and, for modes 1 and 2, the
+/// unpacked size as a u32. A mode 0 payload is stored raw; a mode 1 payload is a zero mask like the file blocks'; a mode
+/// 2 payload is LZ-token coded — not met in the pools examined so far, reported with an empty payload. The records are
+/// found by scanning for the tag and accepting the ones whose header parses; a candidate whose header cannot be read is
+/// retried one byte on, so the scan never stops before the end of the pool. A mode 1 stream that ends exactly at a group
+/// boundary before the declared unpacked size is full leaves the tail of the buffer zero; a stream that ends inside a
+/// group returns the decoded prefix with zero padding and marks the record as partial.
+internal static class SubgroupParser
+{
+ public static List Parse(byte[] pool)
+ {
+ var result = new List();
+ int pos = 0;
+ while (pos + 11 <= pool.Length)
+ {
+ if (pool[pos] != 0x42 || pool[pos + 1] != 0x4C) { pos++; continue; } // "BL"
+ ushort flags = BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 9));
+ if ((flags & FdbFormat.MarkerMask) != FdbFormat.Marker) { pos++; continue; }
+ int q = pos + 11;
+ if ((flags & FdbFormat.FlagGuid) != 0)
+ {
+ if (q + 16 > pool.Length) { pos++; continue; }
+ q += 16;
+ }
+ int sizeClass = flags & 0xF;
+ int width = sizeClass <= 6 ? 2 : sizeClass <= 14 ? 4 : 8;
+ if (q + width > pool.Length) { pos++; continue; }
+ long packed = width switch
+ {
+ 2 => BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(q)),
+ 4 => BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)),
+ _ => unchecked((long)BinaryPrimitives.ReadUInt64LittleEndian(pool.AsSpan(q))),
+ };
+ q += width;
+ var mode = (flags & FdbFormat.FlagZeroMask) != 0 ? PlaBlockMode.ZeroMask
+ : (flags & FdbFormat.FlagLz) != 0 ? PlaBlockMode.Lz
+ : PlaBlockMode.Raw;
+ long unpacked = packed;
+ if (mode != PlaBlockMode.Raw)
+ {
+ if (q + 4 > pool.Length) { pos++; continue; }
+ unpacked = BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q));
+ q += 4;
+ }
+ if (packed < 0 || packed > pool.Length - q || unpacked < 0 || unpacked > (long)pool.Length * 32) { pos++; continue; }
+
+ Guid? objectId = null;
+ if ((flags & FdbFormat.FlagGuid) != 0)
+ objectId = new Guid(pool.AsSpan(pos + 11, 16));
+
+ var data = DecodePayload(pool.AsSpan(q, (int)packed), mode, unpacked, out var partial);
+ result.Add(new PoolSubgroup(pos, BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 6)),
+ pool[pos + 8], flags, mode, objectId, data, partial));
+ pos = q + (int)packed;
+ }
+ return result;
+ }
+
+ private static byte[] DecodePayload(ReadOnlySpan body, PlaBlockMode mode, long unpacked, out bool partial)
+ {
+ if (unpacked > int.MaxValue)
+ {
+ partial = true;
+ return [];
+ }
+ switch (mode)
+ {
+ case PlaBlockMode.Raw:
+ {
+ partial = false;
+ var data = new byte[unpacked];
+ body[..Math.Min(body.Length, data.Length)].CopyTo(data);
+ return data;
+ }
+ case PlaBlockMode.ZeroMask:
+ {
+ // The declared size is the buffer size: a stream that ends exactly at a group boundary leaves the tail
+ // of the buffer zero, but a stream that ends inside a group is a truncated literal — the record is
+ // damaged, its tail is reconstructed with zeros and marked Partial.
+ var data = new byte[unpacked];
+ PayloadDecoder.DecodeZeroMask(body, data, out var end);
+ partial = end == PayloadDecoder.ZeroMaskEnd.MidGroup;
+ return data;
+ }
+ default:
+ partial = true;
+ return []; // Mode 2 (LZ) is not met in the pools examined so far.
+ }
+ }
+}
diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs
index 4cc7ad1..3348319 100644
--- a/src/PlaIO/PlaDocument.cs
+++ b/src/PlaIO/PlaDocument.cs
@@ -142,6 +142,29 @@ public IReadOnlyList FindDirectories()
return found.AsReadOnly();
}
+ /// Finds the compressed storage pools of the object database: every 16-byte compression sign in the file
+ /// whose header and block headers parse and whose declared sizes tile the total. These pools sit in the gaps between
+ /// blocks; the byte ranges of the indexed blocks are excluded, so a sign that a block payload happens to carry is
+ /// never returned as a pool. Computed on each call; the scan reads the file in windows and then the block headers of
+ /// each candidate.
+ public IReadOnlyList FindCompressedPools()
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ // The indexed block payloads are excluded whole: their bytes are data, and a compression sign inside them is
+ // content, not a pool start.
+ var excluded = Blocks.Select(b => (b.PayloadOffset, b.End)).ToList();
+ return PoolDecoder.Find(_stream, Length, excluded)
+ .Select(p => new CompressedPool(this, p.Offset, p.End, (long)p.Total, p.BlockCount))
+ .ToList()
+ .AsReadOnly();
+ }
+
+ internal byte[] DecodePool(long offset)
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ return PoolDecoder.Decode(_stream, offset, Length);
+ }
+
/// Releases the stream unless the document was opened with leaveOpen.
public void Dispose()
{
diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs
new file mode 100644
index 0000000..a21504d
--- /dev/null
+++ b/src/PlaIO/PoolSubgroup.cs
@@ -0,0 +1,49 @@
+namespace PlaIO;
+
+/// One object record inside a compressed pool: a BL-tagged header with the object's class id, a GUID and a
+/// payload decoded from the pool's compression (raw or zero mask). The meaning of the class id and of the payload's
+/// fields is not documented; observed payloads store text as a u32 length followed by UTF-16 little-endian data and
+/// numbers as little-endian doubles and integers.
+public sealed class PoolSubgroup
+{
+ internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMode mode, Guid? objectId, byte[] data, bool partial)
+ {
+ Offset = offset;
+ Kind = kind;
+ Version = version;
+ Flags = flags;
+ Mode = mode;
+ ObjectId = objectId;
+ Data = data;
+ Partial = partial;
+ }
+
+ /// Offset of the subgroup header inside the decoded pool (see ).
+ public long Offset { get; }
+
+ /// The object's class id (the kind field of the BL header). Values seen in real files run from small ids to
+ /// 0xFFDC, which the table of contents uses as its free-block marker.
+ public int Kind { get; }
+
+ /// The header's version byte, 0 or 1.
+ public int Version { get; }
+
+ /// The header's flags: bit 15..14 pair is the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the
+ /// payload modes, the low 4 bits are the packed size class.
+ public int Flags { get; }
+
+ /// The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but its payload is not decoded;
+ /// it has not been met inside the pools examined so far.
+ public PlaBlockMode Mode { get; }
+
+ /// The object's GUID, present when flag 0x20 is set.
+ public Guid? ObjectId { get; }
+
+ /// The decoded payload. For mode 2 subgroups this is empty.
+ public byte[] Data { get; }
+
+ /// Whether decoding was incomplete or unavailable. A truncated zero-mask literal leaves a
+ /// zero-filled tail in . Unsupported mode 2 and payloads too large to decode have empty data.
+ /// A zero-mask stream that ends at a group boundary may have a valid zero-filled tail with this property false.
+ public bool Partial { get; }
+}
diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs
new file mode 100644
index 0000000..0405d5d
--- /dev/null
+++ b/tests/PlaIO.Tests/CompressedPoolTests.cs
@@ -0,0 +1,193 @@
+using System.Buffers.Binary;
+using Xunit;
+
+namespace PlaIO.Tests;
+
+public class CompressedPoolTests
+{
+ private static byte[] PoolOf(params byte[][] payloads) => Synthetic.Fdb(Synthetic.Pool(payloads));
+
+ [Fact]
+ public void AFileWithoutSignsFindsNoPools()
+ {
+ using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.RandomBytes(new Random(1), 4096))));
+ Assert.Empty(document.FindCompressedPools());
+ }
+
+ [Fact]
+ public void OneBigMemCpyBlockDecodesToItsPayload()
+ {
+ var payload = Enumerable.Range(0, 500).Select(i => (byte)(i * 7)).ToArray();
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(payload)));
+ var pools = document.FindCompressedPools();
+ Assert.Equal([1], pools.Select(p => p.BlockCount).ToArray());
+ var pool = Assert.Single(pools);
+ Assert.Equal(payload.Length, pool.TotalDecompressed);
+ Assert.Equal(payload, pool.Decode());
+ }
+
+ [Fact]
+ public void SmallAndBigBlocksDecodeToTheirPayloadsConcatenated()
+ {
+ var first = Synthetic.RandomBytes(new Random(2), 100);
+ var second = Synthetic.RandomBytes(new Random(3), 300);
+ var bytes = Synthetic.Fdb(Synthetic.Pool([first], small: true), Synthetic.Pool([second]));
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pools = document.FindCompressedPools();
+ Assert.Equal(2, pools.Count);
+ Assert.Equal(first, pools[0].Decode());
+ Assert.Equal(second, pools[1].Decode());
+ // The file offsets do not overlap: the second pool starts past the first one's last block.
+ Assert.True(pools[0].End <= pools[1].Offset);
+ }
+
+ [Fact]
+ public void AZeroBlockPoolIsValidAndDecodesToNothing()
+ {
+ var bytes = Synthetic.Fdb(Synthetic.PoolSign(), new byte[16]);
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(document.FindCompressedPools());
+ Assert.Equal(0, pool.TotalDecompressed);
+ Assert.Empty(pool.Decode());
+ }
+
+ [Fact]
+ public void ATruncatedPoolIsNotFound()
+ {
+ var bytes = Synthetic.Pool([Synthetic.RandomBytes(new Random(4), 200)]);
+ using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(bytes[..^32])));
+ Assert.Empty(document.FindCompressedPools());
+ }
+
+ [Fact]
+ public void ABlockSizeMismatchIsNotFound()
+ {
+ // Flip one byte of the block's declared packed size (the pool holds the sign, the two u64 sizes, the u32
+ // compressed size and the flag byte): it stops matching the compressed size the pool lists.
+ var pool = Synthetic.Pool([Synthetic.RandomBytes(new Random(5), 64)]);
+ var corrupt = pool.ToArray();
+ corrupt[16 + 16 + 4 + 1] ^= 0xFF;
+ using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(corrupt)));
+ Assert.Empty(document.FindCompressedPools());
+ }
+
+ [Fact]
+ public void LZBlockThatDoesNotDecodeFindsButDecodeThrows()
+ {
+ // A big block that declares LZ77 coding (bit 0 set, flag 0x47) with an impossible token stream: Find accepts the
+ // shape (the block headers tile the total), Decode fails.
+ var block = new byte[9 + 16];
+ block[0] = 0x47;
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length);
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 16);
+ 0x00FF00FFu.CopyToBlock(block, 9);
+ var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(16), Le64(1), Le32((uint)block.Length), block);
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(document.FindCompressedPools());
+ Assert.Throws(pool.Decode);
+ }
+
+ [Fact]
+ public void ADeclaredTotalPastTheContainerIsStillAccepted()
+ {
+ // The codec is compressing: a small pool can declare a total bigger than the container itself. The block header
+ // parses inside the file and its declared unpacked size tiles the total, so Find accepts the pool even though
+ // the declared total is past the end of the file; only the array limit and the per-block bounds are checked.
+ var block = new byte[9 + 16];
+ block[0] = 0x47;
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length);
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 128);
+ 0x00FF00FFu.CopyToBlock(block, 9);
+ var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(128), Le64(1), Le32((uint)block.Length), block);
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(document.FindCompressedPools());
+ Assert.Equal(128, pool.TotalDecompressed);
+ Assert.True(pool.TotalDecompressed > document.Length);
+ }
+
+ [Fact]
+ public void ASignInsideAnIndexedBlockPayloadIsNotAPool()
+ {
+ // A raw block whose payload carries a compression sign and a parseable pool header: the sign is data, not a
+ // pool start, and Find must not return it.
+ var payload = Synthetic.Pool([Synthetic.RandomBytes(new Random(8), 64)]);
+ var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B);
+ using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(block)));
+ Assert.Empty(document.FindCompressedPools());
+ }
+
+ [Fact]
+ public void APoolInTheGapAfterAnIndexedBlockIsFound()
+ {
+ // The same container with the pool moved out of the block: the block payload is a filler, the pool follows it.
+ var payload = Synthetic.RandomBytes(new Random(9), 64);
+ var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B);
+ var bytes = Synthetic.Fdb(block, Synthetic.Pool([Synthetic.RandomBytes(new Random(10), 64)]));
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(document.FindCompressedPools());
+ Assert.True(pool.Offset >= 32 + payload.Length, $"pool at 0x{pool.Offset:x} overlaps the block");
+ }
+
+ [Fact]
+ public void DecodeAfterDisposeThrows()
+ {
+ var document = PlaDocument.Open(new MemoryStream(PoolOf([1, 2, 3])));
+ var pool = Assert.Single(document.FindCompressedPools());
+ document.Dispose();
+ Assert.Throws(pool.Decode);
+ Assert.Throws(document.FindCompressedPools);
+ }
+
+ [Fact]
+ public void ASignAcrossTheWindowBoundaryIsStillFound()
+ {
+ // Put the sign where the 1 MiB scan window ends eight bytes after its start: the next window must overlap it.
+ var filler = Synthetic.RandomBytes(new Random(6), (1 << 20) - 8);
+ var bytes = Synthetic.Fdb(filler, Synthetic.Pool([Synthetic.RandomBytes(new Random(7), 300)]));
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(document.FindCompressedPools());
+ Assert.Equal(300, pool.TotalDecompressed);
+ Assert.Equal(300, pool.Decode().Length);
+ }
+
+ [SkippableFact]
+ public void EverySampleFindsPoolsAndThePoolsOfTheSmallerOnesDecode()
+ {
+ var corpus = new SampleCorpus();
+ try
+ {
+ foreach (var sample in corpus.Fdb)
+ {
+ var pools = sample.Document.FindCompressedPools();
+ bool decodeAll = sample.Document.Length < 32 << 20;
+ foreach (var pool in decodeAll ? pools : pools.Take(8))
+ Assert.True(pool.Decode().Length == pool.TotalDecompressed,
+ $"{sample.Name}: pool at 0x{pool.Offset:x} does not decode to {pool.TotalDecompressed}");
+ }
+ }
+ finally
+ {
+ corpus.Dispose();
+ }
+ }
+
+ private static byte[] Le32(uint value)
+ {
+ var bytes = new byte[4];
+ System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, value);
+ return bytes;
+ }
+
+ private static byte[] Le64(ulong value)
+ {
+ var bytes = new byte[8];
+ System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(bytes, value);
+ return bytes;
+ }
+}
+
+file static class CopyExtensions
+{
+ public static void CopyToBlock(this uint value, byte[] block, int at) =>
+ System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(at), value);
+}
diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs
new file mode 100644
index 0000000..b9a3cd1
--- /dev/null
+++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs
@@ -0,0 +1,184 @@
+using System.Buffers.Binary;
+using System.Text;
+using Xunit;
+
+namespace PlaIO.Tests;
+
+public class PoolSubgroupTests
+{
+ private static byte[] PoolOf(params byte[][] subgroups) => Synthetic.Fdb(Synthetic.Pool([subgroups.SelectMany(s => s).ToArray()]));
+
+ private static byte[] TextPayload(string text) =>
+ BitConverter.GetBytes((uint)text.Length).Concat(Encoding.Unicode.GetBytes(text)).ToArray();
+
+ [Fact]
+ public void RawSubgroupIsParsedWithKindGuidAndData()
+ {
+ var guid = Guid.NewGuid();
+ var payload = Synthetic.RandomBytes(new Random(1), 100);
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, payload, PlaBlockMode.Raw, guid))));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.Equal(0x009B, subgroup.Kind);
+ Assert.Equal(1, subgroup.Version);
+ Assert.Equal(guid, subgroup.ObjectId);
+ Assert.Equal(PlaBlockMode.Raw, subgroup.Mode);
+ Assert.Equal(payload, subgroup.Data);
+ Assert.Equal(0, subgroup.Offset);
+ }
+
+ [Fact]
+ public void ZeroMaskSubgroupIsDecodedToItsData()
+ {
+ var payload = Synthetic.ZeroMask(Synthetic.RandomBytes(new Random(2), 200));
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0045, payload, PlaBlockMode.ZeroMask, unpacked: 200))));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.Equal(PlaBlockMode.ZeroMask, subgroup.Mode);
+ Assert.Equal(200, subgroup.Data.Length);
+ Assert.Equal(Synthetic.RandomBytes(new Random(2), 200), subgroup.Data);
+ }
+
+ [Fact]
+ public void AZeroMaskStreamShorterThanTheBufferLeavesTheTailZero()
+ {
+ // The declared unpacked size is the buffer size; the stream fills the head and stops.
+ var payload = Synthetic.ZeroMask([1, 2, 3, 4]);
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, payload, PlaBlockMode.ZeroMask, unpacked: 64))));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.Equal(64, subgroup.Data.Length);
+ Assert.Equal(new byte[] { 1, 2, 3, 4 }, subgroup.Data[..4]);
+ Assert.True(subgroup.Data[4..].All(b => b == 0));
+ Assert.False(subgroup.Partial);
+ }
+
+ [Fact]
+ public void ATruncatedLiteralIsMarkedPartialAndKeepsTheProducedBytes()
+ {
+ // The last mask demands more literals than the stream holds: the record is damaged. The subgroup keeps the
+ // bytes the stream produced, the tail is reconstructed with zeros, and Partial marks it.
+ var plain = new byte[] { 1, 2, 3, 4, 5, 6 };
+ var full = Synthetic.ZeroMask(plain); // mask 0x3F, six data bytes, ends at a group boundary
+ // Cut the last two literals off: the mask still demands six, the stream only holds four.
+ var truncated = full[..(full.Length - 2)];
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, truncated, PlaBlockMode.ZeroMask, unpacked: 6))));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.True(subgroup.Partial);
+ Assert.Equal(6, subgroup.Data.Length);
+ Assert.Equal(new byte[] { 1, 2, 3, 4, 0, 0 }, subgroup.Data);
+ }
+
+ [Fact]
+ public void TextPayloadIsReadAsALengthPrefixedUtf16String()
+ {
+ using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, TextPayload("Sample text"), PlaBlockMode.Raw))));
+ var data = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()).Data;
+ int len = BitConverter.ToInt32(data, 0);
+ Assert.Equal("Sample text", Encoding.Unicode.GetString(data, 4, len * 2));
+ }
+
+ [Fact]
+ public void SubgroupsAreParsedInOrderAndOffsetsArePoolRelative()
+ {
+ var first = Synthetic.RandomBytes(new Random(3), 60);
+ var second = Synthetic.RandomBytes(new Random(4), 30);
+ var bytes = PoolOf(
+ Synthetic.Subgroup(0x0003, first, PlaBlockMode.Raw),
+ Synthetic.Subgroup(0x005D, second, PlaBlockMode.ZeroMask, unpacked: 30));
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var subgroups = document.FindCompressedPools()[0].ReadSubgroups();
+ Assert.Equal(2, subgroups.Count);
+ Assert.Equal(0x0003, subgroups[0].Kind);
+ Assert.Equal(first, subgroups[0].Data);
+ Assert.Equal(0x005D, subgroups[1].Kind);
+ Assert.True(subgroups[1].Offset > subgroups[0].Offset);
+ }
+
+ [Fact]
+ public void AMode2SubgroupIsReportedWithoutData()
+ {
+ var bytes = PoolOf(Synthetic.Subgroup(0x0045, Synthetic.RandomBytes(new Random(5), 40), PlaBlockMode.Lz, unpacked: 80));
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.Equal(PlaBlockMode.Lz, subgroup.Mode);
+ Assert.Empty(subgroup.Data);
+ Assert.True(subgroup.Partial);
+ }
+
+ [Fact]
+ public void JunkBetweenSubgroupsIsSkipped()
+ {
+ var filler = Synthetic.RandomBytes(new Random(6), 50);
+ // Junk that starts with "BL" but fails the header checks, right before a real subgroup, in one block.
+ var bytes = PoolOf(new byte[] { 0x42, 0x4C, 0x99 }.Concat(Synthetic.Subgroup(0x0006, filler, PlaBlockMode.Raw)).ToArray());
+ using var document = PlaDocument.Open(new MemoryStream(bytes));
+ var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups());
+ Assert.Equal(0x0006, subgroup.Kind);
+ Assert.Equal(filler, subgroup.Data);
+ }
+
+ [Fact]
+ public void ReadSubgroupsAfterDisposeThrows()
+ {
+ var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0006, [1, 2, 3], PlaBlockMode.Raw))));
+ var pool = Assert.Single(document.FindCompressedPools());
+ document.Dispose();
+ Assert.Throws(pool.ReadSubgroups);
+ }
+
+ [SkippableFact]
+ public void EverySamplePoolParsesItsSubgroups()
+ {
+ var corpus = new SampleCorpus();
+ try
+ {
+ foreach (var sample in corpus.Fdb)
+ foreach (var pool in sample.Document.FindCompressedPools())
+ {
+ var subgroups = pool.ReadSubgroups();
+ Assert.True(subgroups.Count > 0 || pool.TotalDecompressed == 0,
+ $"{sample.Name}: pool at 0x{pool.Offset:x} of {pool.TotalDecompressed} bytes has no subgroups");
+ }
+ }
+ finally
+ {
+ corpus.Dispose();
+ }
+ }
+
+ [SkippableFact]
+ public void SomeSampleSubgroupHoldsALengthPrefixedUtf16String()
+ {
+ // The payloads store strings as [u32 len][utf16le data]; no sample name or content is asserted, only that the
+ // corpus contains at least one such string inside a decoded subgroup payload.
+ var corpus = new SampleCorpus();
+ try
+ {
+ foreach (var sample in corpus.Fdb)
+ foreach (var pool in sample.Document.FindCompressedPools())
+ foreach (var subgroup in pool.ReadSubgroups())
+ {
+ var data = subgroup.Data;
+ for (int i = 0; i + 8 <= data.Length; i++)
+ {
+ int len = BitConverter.ToInt32(data, i);
+ if (len is < 1 or > 500 || i + 4 + len * 2 > data.Length) continue;
+ bool printable = true;
+ for (int k = 0; k < len * 2 && printable; k += 2)
+ {
+ char ch = (char)(data[i + 4 + k] | data[i + 5 + k] << 8);
+ if (ch != 0 && (ch < 0x20 || ch > 0x7E) && (ch < 0x400 || ch > 0x4FF)) printable = false;
+ }
+ if (printable && len >= 2)
+ {
+ Assert.True(true, $"found a length-prefixed utf16 string in {sample.Name}");
+ return;
+ }
+ }
+ }
+ Assert.Fail("no length-prefixed utf16 string found in any pool subgroup of any sample");
+ }
+ finally
+ {
+ corpus.Dispose();
+ }
+ }
+}
diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs
index 75979fd..6bb05b4 100644
--- a/tests/PlaIO.Tests/Synthetic.cs
+++ b/tests/PlaIO.Tests/Synthetic.cs
@@ -125,6 +125,74 @@ public static byte[] RandomBytes(Random random, int length)
/// A container: the 32-byte header, then the parts from offset 32.
public static byte[] Fdb(params byte[][] parts) => Header().Concat(parts.SelectMany(p => p)).ToArray();
+ /// A compressed storage pool: the compression sign, the u64 total and block count, then one u32 compressed
+ /// size plus block per payload. Big memcpy blocks (flag 0x46) unless ; a small block (flag
+ /// 0x44) keeps its sizes in single bytes and needs at most 252 payload bytes. The big-block flag's other bits match
+ /// the real files (0x44/0x46/0x47 with bit 2 and bit 6 set).
+ public static byte[] Pool(byte[][] payloads, bool small = false)
+ {
+ var body = new List();
+ ulong total = 0;
+ foreach (var payload in payloads)
+ {
+ int headerLength = small ? 3 : 9;
+ var block = new byte[headerLength + payload.Length];
+ block[0] = (byte)(small ? 0x44 : 0x46); // bit 0 clear: memcpy; bit 2 and 6 set like the real files
+ if (small)
+ {
+ block[1] = (byte)block.Length;
+ block[2] = (byte)payload.Length;
+ }
+ else
+ {
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length);
+ BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), (uint)payload.Length);
+ }
+ payload.CopyTo(block, headerLength);
+ body.AddRange(Le32((uint)block.Length));
+ body.AddRange(block);
+ total += (ulong)payload.Length;
+ }
+ var head = new byte[32];
+ PoolSign().CopyTo(head);
+ BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(16), total);
+ BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(24), (ulong)payloads.Length);
+ return head.Concat(body).ToArray();
+ }
+
+ /// The compression sign the pools start with.
+ public static byte[] PoolSign() =>
+ [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];
+
+ /// A BL-tagged subgroup as the pools hold them: tag, id, kind, version, flags (marker 0x8080; GUID flag set
+ /// when is given), the GUID, the packed size (u16 for size classes 0..6, u32 for 7..14) and,
+ /// for modes 1 and 2, the u32 unpacked size, then the payload. Mode 1 payloads are zero-mask coded; the declared
+ /// unpacked size may exceed what the stream produces, the tail stays zero. Mode 2 payloads are reported but not
+ /// coded here (the tests only check that the parser reports them with an empty payload).
+ public static byte[] Subgroup(ushort kind, byte[] payload, PlaBlockMode mode = PlaBlockMode.Raw, Guid? guid = null,
+ uint id = 7, byte version = 1, int unpacked = -1)
+ {
+ int sizeClass = SizeClass.Of((ulong)payload.Length);
+ int modeFlag = mode switch { PlaBlockMode.ZeroMask => 0x100, PlaBlockMode.Lz => 0x200, _ => 0 };
+ ushort flags = (ushort)(0x8080 | sizeClass | (guid != null ? 0x20 : 0) | modeFlag);
+ var bytes = new List();
+ bytes.AddRange(System.Text.Encoding.ASCII.GetBytes("BL"));
+ bytes.AddRange(Le32(id));
+ bytes.AddRange(Le16(kind));
+ bytes.Add(version);
+ bytes.AddRange(Le16(flags));
+ if (guid != null) bytes.AddRange(guid.Value.ToByteArray());
+ if (sizeClass <= 6) bytes.AddRange(Le16((ushort)payload.Length));
+ else bytes.AddRange(Le32((uint)payload.Length));
+ if (mode != PlaBlockMode.Raw)
+ {
+ uint declared = unpacked >= 0 ? (uint)unpacked : (uint)payload.Length;
+ bytes.AddRange(Le32(declared));
+ }
+ bytes.AddRange(payload);
+ return bytes.ToArray();
+ }
+
/// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked).
public static byte[] Png(params (string Type, byte[] Data)[] chunks)
{