diff --git a/CHANGELOG.md b/CHANGELOG.md index 420ee9b..3b288b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ API between minor releases. - The position and count of `ObjectDirectoryRecord` are now documented: a byte extent of the decoded global stream, confirmed against the object directories of real files. +- `PlaDocument.FindCompressedPools()` finds pools outside indexed block payloads. `CompressedPool.Decode()` reads + stored and QuickLZ level 1 frames using the public format description and saved-file verification. + ### Changed - The release workflow follows the IOExtend release order: the NuGet packages are pushed before the GitHub release is diff --git a/docs/public-api.md b/docs/public-api.md index 105aba7..cb93656 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -35,6 +35,20 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `PlaStream.ReadExtent(long position, long count)` | The bytes of one extent. Throws `ArgumentOutOfRangeException` for negative or out-of-bounds extents and `InvalidDataException` for extents that reach the unreliable range. | | `PlaStream.ReadExtent(ObjectDirectoryRecord record)` | The bytes addressed by a directory record. Throws `InvalidDataException` if the extent exceeds the stream or reaches the unreliable range. | +## Compressed pools + +Pool framing was verified using saved container bytes. A 16-byte marker is followed by the decoded total and frame count +as little-endian u64 values. Each frame has a u32 size followed by a standalone QuickLZ level 1 frame. The decoder +implements the [public QuickLZ 1.5.0 format](https://github.com/ReSpeak/quicklz/blob/d905415a88aed6dae685bae500025ccc81de3822/Format.md); +independent decoding of saved-file frames verified the correspondence. PlaIO has no runtime dependency on QuickLZ. + +| Member | Use | +|---|---| +| `PlaDocument.FindCompressedPools()` | Finds pool markers outside indexed block payloads and validates frame headers and sizes. Computed on each call. | +| `CompressedPool.Offset` and `End` | File offsets of the pool marker and of the end of the pool. | +| `CompressedPool.TotalDecompressed` and `BlockCount` | Decoded byte count and number of frames in the pool. | +| `CompressedPool.Decode()` | Decoded frame payloads in order. Throws `InvalidDataException` on invalid framing or data and `ObjectDisposedException` after disposing the document. | + ## BIMx file (`.bimx`) | Member | Use | diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs new file mode 100644 index 0000000..2427b8b --- /dev/null +++ b/src/PlaIO/CompressedPool.cs @@ -0,0 +1,35 @@ +using PlaIO.Internal; + +namespace PlaIO; + +/// A pool of QuickLZ frames stored between indexed FDB block payloads. +public sealed class CompressedPool +{ + private readonly PlaDocument _document; + + internal CompressedPool(PlaDocument document, PoolRange range) + { + _document = document; + Offset = range.Offset; + End = range.End; + TotalDecompressed = range.UnpackedSize; + BlockCount = range.FrameCount; + } + + /// The file offset of the pool marker. + public long Offset { get; } + + /// The first file offset after the pool. + public long End { get; } + + /// The declared size of the decoded pool in bytes. + public long TotalDecompressed { get; } + + /// The number of QuickLZ frames in the pool. + public int BlockCount { get; } + + /// Decodes the frames and concatenates their payloads in file order. + /// The pool is malformed or a frame fails to decode. + /// The owning document is disposed. + public byte[] Decode() => _document.DecodePool(Offset); +} diff --git a/src/PlaIO/Internal/PoolReader.cs b/src/PlaIO/Internal/PoolReader.cs new file mode 100644 index 0000000..b2384fc --- /dev/null +++ b/src/PlaIO/Internal/PoolReader.cs @@ -0,0 +1,149 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +internal readonly record struct PoolRange(long Offset, long End, int UnpackedSize, int FrameCount); + +internal static class PoolReader +{ + private static ReadOnlySpan Signature => + [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + private const int PoolHeaderLength = 32; + private const int MinFrameEntryLength = 7; + + private readonly record struct Frame(long PayloadOffset, int PayloadLength, int UnpackedSize, bool IsCompressed); + + public static List Find(Stream stream, long length, IReadOnlyList blocks) + { + var pools = new List(); + var buffer = new byte[64 * 1024]; + long pos = FdbFormat.FirstBlockOffset; + foreach (var block in blocks) + { + pos = ScanRange(stream, length, pos, block.PayloadOffset, buffer, pools); + pos = Math.Max(pos, block.End); + } + ScanRange(stream, length, pos, length, buffer, pools); + return pools; + } + + private static long ScanRange(Stream stream, long length, long pos, long end, byte[] buffer, List pools) + { + while (end - pos >= Signature.Length) + { + long windowStart = pos; + var window = buffer.AsSpan(0, (int)Math.Min(buffer.Length, end - windowStart)); + StreamReads.ReadExactlyAt(stream, windowStart, window, "pool markers"); + pos = windowStart + window.Length - Signature.Length + 1; + int from = 0; + while (window.Length - from >= Signature.Length) + { + int index = window[from..].IndexOf(Signature); + if (index < 0) break; + long offset = windowStart + from + index; + try + { + var (range, _) = ReadLayout(stream, length, offset); + pools.Add(range); + if (range.End >= windowStart + window.Length) + { + pos = range.End; + break; + } + from = (int)(range.End - windowStart); + pos = Math.Max(pos, range.End); + } + catch (InvalidDataException) + { + from += index + 1; + } + } + } + return pos; + } + + public static byte[] Decode(Stream stream, long length, long offset) + { + var (range, frames) = ReadLayout(stream, length, offset); + byte[] decoded = new byte[range.UnpackedSize]; + int written = 0; + foreach (var frame in frames) + { + Span output = decoded.AsSpan(written, frame.UnpackedSize); + if (frame.IsCompressed) + { + byte[] input = new byte[frame.PayloadLength]; + StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, input, "a compressed pool frame"); + QuickLzLevelOne.Decode(input, output); + } + else + { + StreamReads.ReadExactlyAt(stream, frame.PayloadOffset, output, "a stored pool frame"); + } + written += output.Length; + } + return decoded; + } + + private static (PoolRange Range, List Frames) ReadLayout(Stream stream, long length, long offset) + { + if (length - offset < PoolHeaderLength) + throw InvalidPool(offset, "the pool header is cut short"); + + Span header = stackalloc byte[PoolHeaderLength]; + StreamReads.ReadExactlyAt(stream, offset, header, "a pool header"); + if (!header[..Signature.Length].SequenceEqual(Signature)) + throw InvalidPool(offset, "the pool marker is missing"); + + ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header[16..]); + ulong count = BinaryPrimitives.ReadUInt64LittleEndian(header[24..]); + long pos = offset + PoolHeaderLength; + if (total > (ulong)Array.MaxLength || count > int.MaxValue || count > (ulong)((length - pos) / MinFrameEntryLength)) + throw InvalidPool(offset, "the declared pool size or frame count is outside the supported bounds"); + + var frames = new List(); + ulong written = 0; + Span entry = stackalloc byte[13]; + for (int index = 0; index < (int)count; index++) + { + if (length - pos < MinFrameEntryLength) + throw InvalidPool(offset, $"frame {index} has a truncated header"); + + StreamReads.ReadExactlyAt(stream, pos, entry[..MinFrameEntryLength], "a pool frame header"); + uint packed = BinaryPrimitives.ReadUInt32LittleEndian(entry); + byte flags = entry[4]; + if ((flags & 0xFC) != 0x44) + throw InvalidPool(offset, $"frame {index} does not use standalone QuickLZ level 1"); + + bool longHeader = (flags & 2) != 0; + int headerBytes = longHeader ? 9 : 3; + if (packed < headerBytes || packed > length - pos - 4 || packed > Array.MaxLength) + throw InvalidPool(offset, $"frame {index} has an invalid packed size"); + + if (longHeader) + StreamReads.ReadExactlyAt(stream, pos + MinFrameEntryLength, entry[MinFrameEntryLength..], "a long pool frame header"); + uint innerSize = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[5..]) : entry[5]; + uint unpacked = longHeader ? BinaryPrimitives.ReadUInt32LittleEndian(entry[9..]) : entry[6]; + if (innerSize != packed || unpacked > total - written) + throw InvalidPool(offset, $"frame {index} declares inconsistent sizes"); + + bool compressed = (flags & 1) != 0; + int payloadLength = (int)packed - headerBytes; + // The longest match emits 255 bytes for a three-byte reference. + if (compressed ? unpacked > 85L * payloadLength : unpacked != payloadLength) + throw InvalidPool(offset, $"frame {index} cannot produce its declared size"); + + frames.Add(new Frame(pos + 4 + headerBytes, payloadLength, (int)unpacked, compressed)); + written += unpacked; + pos += 4 + (long)packed; + } + + if (written != total) + throw InvalidPool(offset, "the frame sizes do not add up to the declared pool size"); + return (new PoolRange(offset, pos, (int)total, frames.Count), frames); + } + + private static InvalidDataException InvalidPool(long offset, string reason) => + new($"Pool at 0x{offset:x}: {reason}."); +} diff --git a/src/PlaIO/Internal/QuickLzLevelOne.cs b/src/PlaIO/Internal/QuickLzLevelOne.cs new file mode 100644 index 0000000..6bf553a --- /dev/null +++ b/src/PlaIO/Internal/QuickLzLevelOne.cs @@ -0,0 +1,77 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +// QuickLZ 1.5.0 level 1, without streaming. See docs/public-api.md for the public format reference. +internal static class QuickLzLevelOne +{ + public static void Decode(ReadOnlySpan input, Span output) + { + Span matches = stackalloc int[4096]; + matches.Fill(-1); + uint control = 1; + int written = 0; + int nextHash = 0; + bool tail = false; + + while (written < output.Length) + { + if (control == 1) + { + control = TakeWord(ref input, sizeof(uint)); + if ((control & 0x80000000) == 0) + throw new InvalidDataException("A QuickLZ control word has no sentinel bit."); + } + + if (!tail && (control & 1) != 0) + { + uint token = TakeWord(ref input, sizeof(ushort)); + int lengthCode = (int)(token & 15); + int count = lengthCode == 0 ? (int)TakeWord(ref input, sizeof(byte)) : lengthCode + 2; + int start = matches[(int)(token >> 4)]; + if (start < 0 || count < 3 || count > output.Length - written) + throw new InvalidDataException("A QuickLZ reference has an invalid position or length."); + + int matchStart = written; + for (int i = 0; i < count; i++) + output[written++] = output[start + i]; + Register(matches, output, nextHash, matchStart + 1); + nextHash = written; + } + else + { + tail |= output.Length - written <= 10; + output[written++] = (byte)TakeWord(ref input, sizeof(byte)); + if (!tail && nextHash + 3 <= written) + { + Register(matches, output, nextHash, written - 2); + nextHash = written - 2; + } + } + control >>= 1; + } + } + + private static void Register(Span matches, ReadOnlySpan output, int start, int end) + { + for (int pos = start; pos < end; pos++) + { + int word = output[pos] | (output[pos + 1] << 8) | (output[pos + 2] << 16); + matches[(word ^ (word >> 12)) & 0xFFF] = pos; + } + } + + private static uint TakeWord(ref ReadOnlySpan input, int width) + { + if (width > input.Length) + throw new InvalidDataException("A QuickLZ frame ends inside a token."); + uint word = width switch + { + sizeof(uint) => BinaryPrimitives.ReadUInt32LittleEndian(input), + sizeof(ushort) => BinaryPrimitives.ReadUInt16LittleEndian(input), + _ => input[0], + }; + input = input[width..]; + return word; + } +} diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index d426c9a..3185851 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -243,6 +243,19 @@ public IReadOnlyList FindDirectories() return found.AsReadOnly(); } + /// Finds compressed pools whose marker and frame headers parse, outside indexed block payloads. + public IReadOnlyList FindCompressedPools() + { + ObjectDisposedException.ThrowIf(_disposed, this); + return PoolReader.Find(_stream, Length, Blocks).Select(range => new CompressedPool(this, range)).ToList().AsReadOnly(); + } + + internal byte[] DecodePool(long offset) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return PoolReader.Decode(_stream, Length, offset); + } + /// Releases the stream unless the document was opened with leaveOpen. public void Dispose() { diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs new file mode 100644 index 0000000..0935c60 --- /dev/null +++ b/tests/PlaIO.Tests/CompressedPoolTests.cs @@ -0,0 +1,169 @@ +using System.Buffers.Binary; +using Xunit; + +namespace PlaIO.Tests; + +public class CompressedPoolTests +{ + private static ReadOnlySpan PoolMarker => + [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + [Fact] + public void ReadsShortAndLongStoredFrames() + { + byte[] first = "ABC"u8.ToArray(); + byte[] second = Enumerable.Repeat((byte)'Z', 220).ToArray(); + byte[] bytes = Synthetic.Fdb(Pool(((uint)first.Length, Frame(first, first.Length, flags: 0x44)), + ((uint)second.Length, Frame(second, second.Length, flags: 0x46)))); + using var doc = PlaDocument.Open(new MemoryStream(bytes)); + + var pool = Assert.Single(doc.FindCompressedPools()); + Assert.Equal(32, pool.Offset); + Assert.Equal(bytes.Length, pool.End); + Assert.Equal(2, pool.BlockCount); + Assert.Equal(first.Length + second.Length, pool.TotalDecompressed); + Assert.Equal(Synthetic.Concat(first, second), pool.Decode()); + } + + [Fact] + public void DecodesAnOverlappingQuickLzMatchAndLiteralTail() + { + int word = 'A' | ('B' << 8) | ('C' << 16); + ushort token = (ushort)((((word ^ (word >> 12)) & 0xFFF) << 4) | 10); + byte[] tokenBytes = new byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(tokenBytes, token); + byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], "ABCD"u8.ToArray(), tokenBytes, "0123456789"u8.ToArray()); + byte[] expected = "ABCDABCDABCDABCD0123456789"u8.ToArray(); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Pool(((uint)expected.Length, Frame(body, expected.Length, flags: 0x45)))))); + + Assert.Equal(expected, Assert.Single(doc.FindCompressedPools()).Decode()); + } + + [Fact] + public void DecodedPoolCanBeLargerThanTheContainer() + { + byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], new byte[4], [0, 0, 190], new byte[11]); + byte[] bytes = Synthetic.Fdb(Pool((205, Frame(body, 205, flags: 0x47)))); + using var doc = PlaDocument.Open(new MemoryStream(bytes)); + + Assert.True(bytes.Length < 205); + Assert.Equal(new byte[205], Assert.Single(doc.FindCompressedPools()).Decode()); + } + + [Fact] + public void MarkerInsideAnIndexedPayloadIsIgnored() + { + byte[] pool = Pool((3, Frame([1, 2, 3], 3, flags: 0x44))); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Raw, pool), + pool))); + + var block = Assert.Single(doc.Blocks); + Assert.Equal(block.End, Assert.Single(doc.FindCompressedPools()).Offset); + } + + [Fact] + public void FindsMarkerAcrossTheScanWindowBoundary() + { + byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Pool()); + using var doc = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(doc.FindCompressedPools()); + Assert.Equal(bytes.Length, pool.End); + Assert.Empty(pool.Decode()); + } + + [Fact] + public void RejectsATruncatedFrameAtEndOfFile() + { + byte[] frame = Frame([1, 2, 3], 3, flags: 0x44); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))[..^1]))); + + Assert.Empty(doc.FindCompressedPools()); + } + + [Fact] + public void APoolWithInconsistentLengthsDoesNotHideAValidPool() + { + byte[] frame = Frame([1, 2, 3], 3, flags: 0x44); + byte[] bytes = Synthetic.Fdb(Pool((4, frame)), Pool((3, frame))); + using var doc = PlaDocument.Open(new MemoryStream(bytes)); + + var pool = Assert.Single(doc.FindCompressedPools()); + Assert.Equal(bytes.Length, pool.End); + Assert.Equal(new byte[] { 1, 2, 3 }, pool.Decode()); + } + + [Fact] + public void DecodeRejectsAnUninitializedMatch() + { + byte[] frame = Frame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))))); + var pool = Assert.Single(doc.FindCompressedPools()); + Assert.Throws(pool.Decode); + } + + [Fact] + public void DecodeAfterDisposeThrows() + { + var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool()))); + var pool = Assert.Single(doc.FindCompressedPools()); + doc.Dispose(); + + Assert.Throws(pool.Decode); + Assert.Throws(doc.FindCompressedPools); + } + + [SkippableFact] + public void SavedFilePoolFramesDecodeToTheirDeclaredSize() + { + using var corpus = new SampleCorpus(); + int decoded = 0; + foreach (var sample in corpus.Fdb) + { + foreach (var pool in sample.Document.FindCompressedPools()) + { + Assert.Equal(pool.TotalDecompressed, pool.Decode().LongLength); + decoded++; + } + } + if (decoded == 0) throw TestFiles.Missing("FDB fixture with QuickLZ pools"); + } + + private static byte[] Pool(params (uint Size, byte[] Frame)[] frames) + { + byte[] header = new byte[32]; + PoolMarker.CopyTo(header); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), (ulong)frames.Sum(f => (long)f.Size)); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(24), (ulong)frames.Length); + using var result = new MemoryStream(); + result.Write(header); + Span size = stackalloc byte[4]; + foreach (var (_, frame) in frames) + { + BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)frame.Length); + result.Write(size); + result.Write(frame); + } + return result.ToArray(); + } + + private static byte[] Frame(byte[] body, int size, byte flags) + { + int header = (flags & 2) == 0 ? 3 : 9; + byte[] frame = new byte[header + body.Length]; + frame[0] = flags; + if (header == 3) + { + frame[1] = checked((byte)frame.Length); + frame[2] = checked((byte)size); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(1), (uint)frame.Length); + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(5), (uint)size); + } + body.CopyTo(frame, header); + return frame; + } +}