diff --git a/CHANGELOG.md b/CHANGELOG.md
index 420ee9b..3b288b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,9 @@ API between minor releases.
- The position and count of `ObjectDirectoryRecord` are now documented: a byte extent of the decoded global stream,
confirmed against the object directories of real files.
+- `PlaDocument.FindCompressedPools()` finds pools outside indexed block payloads. `CompressedPool.Decode()` reads
+ stored and QuickLZ level 1 frames using the public format description and saved-file verification.
+
### Changed
- The release workflow follows the IOExtend release order: the NuGet packages are pushed before the GitHub release is
diff --git a/docs/public-api.md b/docs/public-api.md
index 105aba7..cb93656 100644
--- a/docs/public-api.md
+++ b/docs/public-api.md
@@ -35,6 +35,20 @@ The `PlaIO` package is the supported surface listed here; every public type is i
| `PlaStream.ReadExtent(long position, long count)` | The bytes of one extent. Throws `ArgumentOutOfRangeException` for negative or out-of-bounds extents and `InvalidDataException` for extents that reach the unreliable range. |
| `PlaStream.ReadExtent(ObjectDirectoryRecord record)` | The bytes addressed by a directory record. Throws `InvalidDataException` if the extent exceeds the stream or reaches the unreliable range. |
+## Compressed pools
+
+Pool framing was verified using saved container bytes. A 16-byte marker is followed by the decoded total and frame count
+as little-endian u64 values. Each frame has a u32 size followed by a standalone QuickLZ level 1 frame. The decoder
+implements the [public QuickLZ 1.5.0 format](https://github.com/ReSpeak/quicklz/blob/d905415a88aed6dae685bae500025ccc81de3822/Format.md);
+independent decoding of saved-file frames verified the correspondence. PlaIO has no runtime dependency on QuickLZ.
+
+| Member | Use |
+|---|---|
+| `PlaDocument.FindCompressedPools()` | Finds pool markers outside indexed block payloads and validates frame headers and sizes. Computed on each call. |
+| `CompressedPool.Offset` and `End` | File offsets of the pool marker and of the end of the pool. |
+| `CompressedPool.TotalDecompressed` and `BlockCount` | Decoded byte count and number of frames in the pool. |
+| `CompressedPool.Decode()` | Decoded frame payloads in order. Throws `InvalidDataException` on invalid framing or data and `ObjectDisposedException` after disposing the document. |
+
## BIMx file (`.bimx`)
| Member | Use |
diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs
new file mode 100644
index 0000000..2427b8b
--- /dev/null
+++ b/src/PlaIO/CompressedPool.cs
@@ -0,0 +1,35 @@
+using PlaIO.Internal;
+
+namespace PlaIO;
+
+/// A pool of QuickLZ frames stored between indexed FDB block payloads.
+public sealed class CompressedPool
+{
+ private readonly PlaDocument _document;
+
+ internal CompressedPool(PlaDocument document, PoolRange range)
+ {
+ _document = document;
+ Offset = range.Offset;
+ End = range.End;
+ TotalDecompressed = range.UnpackedSize;
+ BlockCount = range.FrameCount;
+ }
+
+ /// The file offset of the pool marker.
+ public long Offset { get; }
+
+ /// The first file offset after the pool.
+ public long End { get; }
+
+ /// The declared size of the decoded pool in bytes.
+ public long TotalDecompressed { get; }
+
+ /// The number of QuickLZ frames in the pool.
+ public int BlockCount { get; }
+
+ /// Decodes the frames and concatenates their payloads in file order.
+ /// The pool is malformed or a frame fails to decode.
+ /// The owning document is disposed.
+ public byte[] Decode() => _document.DecodePool(Offset);
+}
diff --git a/src/PlaIO/Internal/PoolReader.cs b/src/PlaIO/Internal/PoolReader.cs
new file mode 100644
index 0000000..b2384fc
--- /dev/null
+++ b/src/PlaIO/Internal/PoolReader.cs
@@ -0,0 +1,149 @@
+using System.Buffers.Binary;
+
+namespace PlaIO.Internal;
+
+internal readonly record struct PoolRange(long Offset, long End, int UnpackedSize, int FrameCount);
+
+internal static class PoolReader
+{
+ private static ReadOnlySpan Signature =>
+ [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];
+
+ private const int PoolHeaderLength = 32;
+ private const int MinFrameEntryLength = 7;
+
+ private readonly record struct Frame(long PayloadOffset, int PayloadLength, int UnpackedSize, bool IsCompressed);
+
+ public static List Find(Stream stream, long length, IReadOnlyList blocks)
+ {
+ var pools = new List();
+ var buffer = new byte[64 * 1024];
+ long pos = FdbFormat.FirstBlockOffset;
+ foreach (var block in blocks)
+ {
+ pos = ScanRange(stream, length, pos, block.PayloadOffset, buffer, pools);
+ pos = Math.Max(pos, block.End);
+ }
+ ScanRange(stream, length, pos, length, buffer, pools);
+ return pools;
+ }
+
+ private static long ScanRange(Stream stream, long length, long pos, long end, byte[] buffer, List pools)
+ {
+ while (end - pos >= Signature.Length)
+ {
+ long windowStart = pos;
+ var window = buffer.AsSpan(0, (int)Math.Min(buffer.Length, end - windowStart));
+ StreamReads.ReadExactlyAt(stream, windowStart, window, "pool markers");
+ pos = windowStart + window.Length - Signature.Length + 1;
+ int from = 0;
+ while (window.Length - from >= Signature.Length)
+ {
+ int index = window[from..].IndexOf(Signature);
+ if (index < 0) break;
+ long offset = windowStart + from + index;
+ try
+ {
+ var (range, _) = ReadLayout(stream, length, offset);
+ pools.Add(range);
+ if (range.End >= windowStart + window.Length)
+ {
+ pos = range.End;
+ break;
+ }
+ from = (int)(range.End - windowStart);
+ pos = Math.Max(pos, range.End);
+ }
+ catch (InvalidDataException)
+ {
+ from += index + 1;
+ }
+ }
+ }
+ return pos;
+ }
+
+ public static byte[] Decode(Stream stream, long length, long offset)
+ {
+ var (range, frames) = ReadLayout(stream, length, offset);
+ byte[] decoded = new byte[range.UnpackedSize];
+ int written = 0;
+ foreach (var frame in frames)
+ {
+ Span output = decoded.AsSpan(written, frame.UnpackedSize);
+ if (frame.IsCompressed)
+ {
+ byte[] input = new byte[frame.PayloadLength];
+ StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, input, "a compressed pool frame");
+ QuickLzLevelOne.Decode(input, output);
+ }
+ else
+ {
+ StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, output, "a stored pool frame");
+ }
+ written += output.Length;
+ }
+ return decoded;
+ }
+
+ private static (PoolRange Range, List Frames) ReadLayout(Stream stream, long length, long offset)
+ {
+ if (length - offset < PoolHeaderLength)
+ throw InvalidPool(offset, "the pool header is cut short");
+
+ Span header = stackalloc byte[PoolHeaderLength];
+ StreamReads.ReadExactlyAt(stream, offset, header, "a pool header");
+ if (!header[..Signature.Length].SequenceEqual(Signature))
+ throw InvalidPool(offset, "the pool marker is missing");
+
+ ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header[16..]);
+ ulong count = BinaryPrimitives.ReadUInt64LittleEndian(header[24..]);
+ long pos = offset + PoolHeaderLength;
+ if (total > (ulong)Array.MaxLength || count > int.MaxValue || count > (ulong)((length - pos) / MinFrameEntryLength))
+ throw InvalidPool(offset, "the declared pool size or frame count is outside the supported bounds");
+
+ var frames = new List();
+ ulong written = 0;
+ Span entry = stackalloc byte[13];
+ for (int index = 0; index < (int)count; index++)
+ {
+ if (length - pos < MinFrameEntryLength)
+ throw InvalidPool(offset, $"frame {index} has a truncated header");
+
+ StreamReads.ReadExactlyAt(stream, pos, entry[..MinFrameEntryLength], "a pool frame header");
+ uint packed = BinaryPrimitives.ReadUInt32LittleEndian(entry);
+ byte flags = entry[4];
+ if ((flags & 0xFC) != 0x44)
+ throw InvalidPool(offset, $"frame {index} does not use standalone QuickLZ level 1");
+
+ bool longHeader = (flags & 2) != 0;
+ int headerBytes = longHeader ? 9 : 3;
+ if (packed < headerBytes || packed > length - pos - 4 || packed > Array.MaxLength)
+ throw InvalidPool(offset, $"frame {index} has an invalid packed size");
+
+ if (longHeader)
+ StreamReads.ReadExactlyAt(stream, pos + MinFrameEntryLength, entry[MinFrameEntryLength..], "a long pool frame header");
+ uint innerSize = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[5..]) : entry[5];
+ uint unpacked = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[9..]) : entry[6];
+ if (innerSize != packed || unpacked > total - written)
+ throw InvalidPool(offset, $"frame {index} declares inconsistent sizes");
+
+ bool compressed = (flags & 1) != 0;
+ int payloadLength = (int)packed - headerBytes;
+ // The longest match emits 255 bytes for a three-byte reference.
+ if (compressed ? unpacked > 85L * payloadLength : unpacked != payloadLength)
+ throw InvalidPool(offset, $"frame {index} cannot produce its declared size");
+
+ frames.Add(new Frame(pos + 4 + headerBytes, payloadLength, (int)unpacked, compressed));
+ written += unpacked;
+ pos += 4 + (long)packed;
+ }
+
+ if (written != total)
+ throw InvalidPool(offset, "the frame sizes do not add up to the declared pool size");
+ return (new PoolRange(offset, pos, (int)total, frames.Count), frames);
+ }
+
+ private static InvalidDataException InvalidPool(long offset, string reason) =>
+ new($"Pool at 0x{offset:x}: {reason}.");
+}
diff --git a/src/PlaIO/Internal/QuickLzLevelOne.cs b/src/PlaIO/Internal/QuickLzLevelOne.cs
new file mode 100644
index 0000000..6bf553a
--- /dev/null
+++ b/src/PlaIO/Internal/QuickLzLevelOne.cs
@@ -0,0 +1,77 @@
+using System.Buffers.Binary;
+
+namespace PlaIO.Internal;
+
+// QuickLZ 1.5.0 level 1, without streaming. See docs/public-api.md for the public format reference.
+internal static class QuickLzLevelOne
+{
+ public static void Decode(ReadOnlySpan input, Span output)
+ {
+ Span matches = stackalloc int[4096];
+ matches.Fill(-1);
+ uint control = 1;
+ int written = 0;
+ int nextHash = 0;
+ bool tail = false;
+
+ while (written < output.Length)
+ {
+ if (control == 1)
+ {
+ control = TakeWord(ref input, sizeof(uint));
+ if ((control & 0x80000000) == 0)
+ throw new InvalidDataException("A QuickLZ control word has no sentinel bit.");
+ }
+
+ if (!tail && (control & 1) != 0)
+ {
+ uint token = TakeWord(ref input, sizeof(ushort));
+ int lengthCode = (int)(token & 15);
+ int count = lengthCode == 0 ? (int)TakeWord(ref input, sizeof(byte)) : lengthCode + 2;
+ int start = matches[(int)(token >> 4)];
+ if (start < 0 || count < 3 || count > output.Length - written)
+ throw new InvalidDataException("A QuickLZ reference has an invalid position or length.");
+
+ int matchStart = written;
+ for (int i = 0; i < count; i++)
+ output[written++] = output[start + i];
+ Register(matches, output, nextHash, matchStart + 1);
+ nextHash = written;
+ }
+ else
+ {
+ tail |= output.Length - written <= 10;
+ output[written++] = (byte)TakeWord(ref input, sizeof(byte));
+ if (!tail && nextHash + 3 <= written)
+ {
+ Register(matches, output, nextHash, written - 2);
+ nextHash = written - 2;
+ }
+ }
+ control >>= 1;
+ }
+ }
+
+ private static void Register(Span matches, ReadOnlySpan output, int start, int end)
+ {
+ for (int pos = start; pos < end; pos++)
+ {
+ int word = output[pos] | (output[pos + 1] << 8) | (output[pos + 2] << 16);
+ matches[(word ^ (word >> 12)) & 0xFFF] = pos;
+ }
+ }
+
+ private static uint TakeWord(ref ReadOnlySpan input, int width)
+ {
+ if (width > input.Length)
+ throw new InvalidDataException("A QuickLZ frame ends inside a token.");
+ uint word = width switch
+ {
+ sizeof(uint) => BinaryPrimitives.ReadUInt32LittleEndian(input),
+ sizeof(ushort) => BinaryPrimitives.ReadUInt16LittleEndian(input),
+ _ => input[0],
+ };
+ input = input[width..];
+ return word;
+ }
+}
diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs
index d426c9a..3185851 100644
--- a/src/PlaIO/PlaDocument.cs
+++ b/src/PlaIO/PlaDocument.cs
@@ -243,6 +243,19 @@ public IReadOnlyList FindDirectories()
return found.AsReadOnly();
}
+ /// Finds compressed pools whose marker and frame headers parse, outside indexed block payloads.
+ public IReadOnlyList FindCompressedPools()
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ return PoolReader.Find(_stream, Length, Blocks).Select(range => new CompressedPool(this, range)).ToList().AsReadOnly();
+ }
+
+ internal byte[] DecodePool(long offset)
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ return PoolReader.Decode(_stream, Length, offset);
+ }
+
/// Releases the stream unless the document was opened with leaveOpen.
public void Dispose()
{
diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs
new file mode 100644
index 0000000..0935c60
--- /dev/null
+++ b/tests/PlaIO.Tests/CompressedPoolTests.cs
@@ -0,0 +1,169 @@
+using System.Buffers.Binary;
+using Xunit;
+
+namespace PlaIO.Tests;
+
+public class CompressedPoolTests
+{
+ private static ReadOnlySpan PoolMarker =>
+ [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06];
+
+ [Fact]
+ public void ReadsShortAndLongStoredFrames()
+ {
+ byte[] first = "ABC"u8.ToArray();
+ byte[] second = Enumerable.Repeat((byte)'Z', 220).ToArray();
+ byte[] bytes = Synthetic.Fdb(Pool(((uint)first.Length, Frame(first, first.Length, flags: 0x44)),
+ ((uint)second.Length, Frame(second, second.Length, flags: 0x46))));
+ using var doc = PlaDocument.Open(new MemoryStream(bytes));
+
+ var pool = Assert.Single(doc.FindCompressedPools());
+ Assert.Equal(32, pool.Offset);
+ Assert.Equal(bytes.Length, pool.End);
+ Assert.Equal(2, pool.BlockCount);
+ Assert.Equal(first.Length + second.Length, pool.TotalDecompressed);
+ Assert.Equal(Synthetic.Concat(first, second), pool.Decode());
+ }
+
+ [Fact]
+ public void DecodesAnOverlappingQuickLzMatchAndLiteralTail()
+ {
+ int word = 'A' | ('B' << 8) | ('C' << 16);
+ ushort token = (ushort)((((word ^ (word >> 12)) & 0xFFF) << 4) | 10);
+ byte[] tokenBytes = new byte[2];
+ BinaryPrimitives.WriteUInt16LittleEndian(tokenBytes, token);
+ byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], "ABCD"u8.ToArray(), tokenBytes, "0123456789"u8.ToArray());
+ byte[] expected = "ABCDABCDABCDABCD0123456789"u8.ToArray();
+ using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(
+ Pool(((uint)expected.Length, Frame(body, expected.Length, flags: 0x45))))));
+
+ Assert.Equal(expected, Assert.Single(doc.FindCompressedPools()).Decode());
+ }
+
+ [Fact]
+ public void DecodedPoolCanBeLargerThanTheContainer()
+ {
+ byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], new byte[4], [0, 0, 190], new byte[11]);
+ byte[] bytes = Synthetic.Fdb(Pool((205, Frame(body, 205, flags: 0x47))));
+ using var doc = PlaDocument.Open(new MemoryStream(bytes));
+
+ Assert.True(bytes.Length < 205);
+ Assert.Equal(new byte[205], Assert.Single(doc.FindCompressedPools()).Decode());
+ }
+
+ [Fact]
+ public void MarkerInsideAnIndexedPayloadIsIgnored()
+ {
+ byte[] pool = Pool((3, Frame([1, 2, 3], 3, flags: 0x44)));
+ using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(
+ Synthetic.Block(PlaBlockMode.Raw, pool),
+ pool)));
+
+ var block = Assert.Single(doc.Blocks);
+ Assert.Equal(block.End, Assert.Single(doc.FindCompressedPools()).Offset);
+ }
+
+ [Fact]
+ public void FindsMarkerAcrossTheScanWindowBoundary()
+ {
+ byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Pool());
+ using var doc = PlaDocument.Open(new MemoryStream(bytes));
+ var pool = Assert.Single(doc.FindCompressedPools());
+ Assert.Equal(bytes.Length, pool.End);
+ Assert.Empty(pool.Decode());
+ }
+
+ [Fact]
+ public void RejectsATruncatedFrameAtEndOfFile()
+ {
+ byte[] frame = Frame([1, 2, 3], 3, flags: 0x44);
+ using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))[..^1])));
+
+ Assert.Empty(doc.FindCompressedPools());
+ }
+
+ [Fact]
+ public void APoolWithInconsistentLengthsDoesNotHideAValidPool()
+ {
+ byte[] frame = Frame([1, 2, 3], 3, flags: 0x44);
+ byte[] bytes = Synthetic.Fdb(Pool((4, frame)), Pool((3, frame)));
+ using var doc = PlaDocument.Open(new MemoryStream(bytes));
+
+ var pool = Assert.Single(doc.FindCompressedPools());
+ Assert.Equal(bytes.Length, pool.End);
+ Assert.Equal(new byte[] { 1, 2, 3 }, pool.Decode());
+ }
+
+ [Fact]
+ public void DecodeRejectsAnUninitializedMatch()
+ {
+ byte[] frame = Frame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45);
+ using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame)))));
+ var pool = Assert.Single(doc.FindCompressedPools());
+ Assert.Throws(pool.Decode);
+ }
+
+ [Fact]
+ public void DecodeAfterDisposeThrows()
+ {
+ var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool())));
+ var pool = Assert.Single(doc.FindCompressedPools());
+ doc.Dispose();
+
+ Assert.Throws(pool.Decode);
+ Assert.Throws(doc.FindCompressedPools);
+ }
+
+ [SkippableFact]
+ public void SavedFilePoolFramesDecodeToTheirDeclaredSize()
+ {
+ using var corpus = new SampleCorpus();
+ int decoded = 0;
+ foreach (var sample in corpus.Fdb)
+ {
+ foreach (var pool in sample.Document.FindCompressedPools())
+ {
+ Assert.Equal(pool.TotalDecompressed, pool.Decode().LongLength);
+ decoded++;
+ }
+ }
+ if (decoded == 0) throw TestFiles.Missing("FDB fixture with QuickLZ pools");
+ }
+
+ private static byte[] Pool(params (uint Size, byte[] Frame)[] frames)
+ {
+ byte[] header = new byte[32];
+ PoolMarker.CopyTo(header);
+ BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), (ulong)frames.Sum(f => (long)f.Size));
+ BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(24), (ulong)frames.Length);
+ using var result = new MemoryStream();
+ result.Write(header);
+ Span size = stackalloc byte[4];
+ foreach (var (_, frame) in frames)
+ {
+ BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)frame.Length);
+ result.Write(size);
+ result.Write(frame);
+ }
+ return result.ToArray();
+ }
+
+ private static byte[] Frame(byte[] body, int size, byte flags)
+ {
+ int header = (flags & 2) == 0 ? 3 : 9;
+ byte[] frame = new byte[header + body.Length];
+ frame[0] = flags;
+ if (header == 3)
+ {
+ frame[1] = checked((byte)frame.Length);
+ frame[2] = checked((byte)size);
+ }
+ else
+ {
+ BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(1), (uint)frame.Length);
+ BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(5), (uint)size);
+ }
+ body.CopyTo(frame, header);
+ return frame;
+ }
+}