From 4be5694b301488dfcfd1b3d36e1fb87ba166abb3 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:44:46 +0300 Subject: [PATCH] Read BL subgroups from decoded pools --- CHANGELOG.md | 2 + docs/public-api.md | 13 ++++ src/PlaIO/CompressedPool.cs | 5 ++ src/PlaIO/Internal/BlockHeaderParser.cs | 7 +- src/PlaIO/Internal/PayloadDecoder.cs | 20 +++++- src/PlaIO/Internal/PoolSubgroupReader.cs | 43 +++++++++++ src/PlaIO/PoolSubgroup.cs | 42 +++++++++++ tests/PlaIO.Tests/CompressedPoolTests.cs | 65 ++++------------- tests/PlaIO.Tests/PoolSubgroupTests.cs | 92 ++++++++++++++++++++++++ tests/PlaIO.Tests/Synthetic.cs | 40 +++++++++++ 10 files changed, 271 insertions(+), 58 deletions(-) create mode 100644 src/PlaIO/Internal/PoolSubgroupReader.cs create mode 100644 src/PlaIO/PoolSubgroup.cs create mode 100644 tests/PlaIO.Tests/PoolSubgroupTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f2702..fd309cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ API between minor releases. - `PlaDocument.FindCompressedPools()` finds pools outside indexed block payloads. `CompressedPool.Decode()` reads stored and QuickLZ level 1 frames using the public format description and saved-file verification. +- `CompressedPool.ReadSubgroups()` reads BL-tagged records from decoded pools and reports incomplete payloads. + ## [0.1.0] - 2026-11-12 ### Added diff --git a/docs/public-api.md b/docs/public-api.md index 7dc2cec..d82178e 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -44,6 +44,19 @@ independent decoding of saved-file frames verified the correspondence. PlaIO has | `CompressedPool.TotalDecompressed` and `BlockCount` | Decoded byte count and number of frames in the pool. | | `CompressedPool.Decode()` | Decoded frame payloads in order. Throws `InvalidDataException` on invalid framing or data and `ObjectDisposedException` after disposing the document. | +### Pool subgroups + +The subgroup header uses the FDB size field widths and may carry a GUID. It does not contain the file-level extra field. +Offsets are relative to the decoded pool. The scanner retries an unmatched BL candidate at the next byte. + +| Member | Use | +|---|---| +| `CompressedPool.ReadSubgroups()` | Decodes the pool and returns the recognised BL-tagged records in pool order. | +| `PoolSubgroup.Offset` | Start of the subgroup header in the decoded pool. | +| `PoolSubgroup.Kind`, `Version`, `Flags` and `ObjectId` | Raw header metadata and the optional GUID. | +| `PoolSubgroup.Mode` and `Data` | Raw or zero-mask payload bytes. Mode 2 is reported with empty data. | +| `PoolSubgroup.Partial` | True for unavailable mode 2 data or an incomplete zero-mask literal. A zero-mask stream ending at a group boundary can leave valid zero padding. | + ## BIMx file (`.bimx`) | Member | Use | diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs index 2427b8b..6e8e616 100644 --- a/src/PlaIO/CompressedPool.cs +++ b/src/PlaIO/CompressedPool.cs @@ -32,4 +32,9 @@ internal CompressedPool(PlaDocument document, PoolRange range) /// The pool is malformed or a frame fails to decode. /// The owning document is disposed. public byte[] Decode() => _document.DecodePool(Offset); + + /// Decodes the pool and reads BL-tagged records in pool order. + /// The pool does not decode. + /// The owning document is disposed. + public IReadOnlyList ReadSubgroups() => PoolSubgroupReader.Read(Decode()); } diff --git a/src/PlaIO/Internal/BlockHeaderParser.cs b/src/PlaIO/Internal/BlockHeaderParser.cs index d0f5198..225fbb4 100644 --- a/src/PlaIO/Internal/BlockHeaderParser.cs +++ b/src/PlaIO/Internal/BlockHeaderParser.cs @@ -8,7 +8,7 @@ internal static class BlockHeaderParser /// Parses a block header that starts at [0], found at file offset . /// The span may end before a full header near the end of the file. Returns false at the first rule that fails; never /// throws. - public static bool TryParse(ReadOnlySpan at, long offset, long fileLength, [NotNullWhen(true)] out PlaBlock? block) + public static bool TryParse(ReadOnlySpan at, long offset, long fileLength, [NotNullWhen(true)] out PlaBlock? block, bool poolSubgroup = false) { block = null; if (at.Length < 11) return false; @@ -39,7 +39,7 @@ public static bool TryParse(ReadOnlySpan at, long offset, long fileLength, }; p += width; if (packed > long.MaxValue) return false; - if (SizeClass.Of(packed) != sizeClass) return false; + if (!poolSubgroup && SizeClass.Of(packed) != sizeClass) return false; var mode = (flags & FdbFormat.FlagZeroMask) != 0 ? PlaBlockMode.ZeroMask : (flags & FdbFormat.FlagLz) != 0 ? PlaBlockMode.Lz @@ -53,7 +53,8 @@ public static bool TryParse(ReadOnlySpan at, long offset, long fileLength, } uint? extra = null; - if ((flags & FdbFormat.FlagExtra) != 0) + // Pool subgroups use the file header's size widths but have no separate extra field. + if (!poolSubgroup && (flags & FdbFormat.FlagExtra) != 0) { if (at.Length < p + 4) return false; extra = BinaryPrimitives.ReadUInt32LittleEndian(at[p..]); diff --git a/src/PlaIO/Internal/PayloadDecoder.cs b/src/PlaIO/Internal/PayloadDecoder.cs index 4d9e88d..7898d81 100644 --- a/src/PlaIO/Internal/PayloadDecoder.cs +++ b/src/PlaIO/Internal/PayloadDecoder.cs @@ -8,10 +8,20 @@ internal static class PayloadDecoder /// left after that are allowed. public static int DecodeZeroMask(ReadOnlySpan src, Span dst) { - int consumed = 0, written = 0; + int consumed = DecodeZeroMask(src, dst, out int written, out _); + if (written != dst.Length) throw Short(consumed, src.Length, written, dst.Length); + return consumed; + } + + /// Decodes the available data and reports whether input ended during a literal group. + public static int DecodeZeroMask(ReadOnlySpan src, Span dst, out int written, out bool truncatedLiteral) + { + int consumed = 0; + written = 0; + truncatedLiteral = false; while (written < dst.Length) { - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) break; int mask = src[consumed++]; for (int bit = 0; bit < 8 && written < dst.Length; bit++) { @@ -20,7 +30,11 @@ public static int DecodeZeroMask(ReadOnlySpan src, Span dst) dst[written++] = 0; continue; } - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) + { + truncatedLiteral = true; + return consumed; + } dst[written++] = src[consumed++]; } } diff --git a/src/PlaIO/Internal/PoolSubgroupReader.cs b/src/PlaIO/Internal/PoolSubgroupReader.cs new file mode 100644 index 0000000..3578e16 --- /dev/null +++ b/src/PlaIO/Internal/PoolSubgroupReader.cs @@ -0,0 +1,43 @@ +namespace PlaIO.Internal; + +internal static class PoolSubgroupReader +{ + public static IReadOnlyList Read(byte[] pool) + { + var subgroups = new List(); + int pos = 0; + while (pool.Length - pos >= FdbFormat.MinBlockHeaderLength) + { + int next = pool.AsSpan(pos).IndexOf("BL"u8); + if (next < 0) break; + pos += next; + if (!BlockHeaderParser.TryParse(pool.AsSpan(pos), pos, pool.Length, out var header, poolSubgroup: true)) + { + pos++; + continue; + } + + ReadOnlySpan payload = pool.AsSpan((int)header.PayloadOffset, (int)header.PackedSize); + byte[] data; + bool partial; + switch (header.Mode) + { + case PlaBlockMode.Raw: + data = payload.ToArray(); + partial = false; + break; + case PlaBlockMode.ZeroMask: + data = new byte[header.UnpackedSize]; + PayloadDecoder.DecodeZeroMask(payload, data, out _, out partial); + break; + default: + data = []; + partial = true; + break; + } + subgroups.Add(new PoolSubgroup(header, data, partial)); + pos = (int)header.End; + } + return subgroups.AsReadOnly(); + } +} diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs new file mode 100644 index 0000000..09144c8 --- /dev/null +++ b/src/PlaIO/PoolSubgroup.cs @@ -0,0 +1,42 @@ +namespace PlaIO; + +/// A BL-tagged record in a decoded compressed pool. +public sealed class PoolSubgroup +{ + internal PoolSubgroup(PlaBlock header, byte[] data, bool partial) + { + Offset = header.Offset; + Kind = header.Kind; + Version = header.Version; + Flags = header.Flags; + Mode = header.Mode; + ObjectId = header.Guid; + Data = data; + Partial = partial; + } + + /// The offset of the header in the decoded pool. + public long Offset { get; } + + /// The raw class number from the header. + public int Kind { get; } + + /// The version byte. + public int Version { get; } + + /// The raw header flags. + public int Flags { get; } + + /// The storage mode of the payload. + public PlaBlockMode Mode { get; } + + /// The object GUID when present in the header. + public Guid? ObjectId { get; } + + /// The decoded payload. Mode 2 returns empty data pending a supported decoder. + public byte[] Data { get; } + + /// True when decoding is unavailable or the zero-mask stream ends inside a literal group. + /// A group-boundary ending can have a valid zero-filled tail with this property false. + public bool Partial { get; } +} diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs index 0935c60..b562c43 100644 --- a/tests/PlaIO.Tests/CompressedPoolTests.cs +++ b/tests/PlaIO.Tests/CompressedPoolTests.cs @@ -5,16 +5,13 @@ namespace PlaIO.Tests; public class CompressedPoolTests { - private static ReadOnlySpan PoolMarker => - [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; - [Fact] public void ReadsShortAndLongStoredFrames() { byte[] first = "ABC"u8.ToArray(); byte[] second = Enumerable.Repeat((byte)'Z', 220).ToArray(); - byte[] bytes = Synthetic.Fdb(Pool(((uint)first.Length, Frame(first, first.Length, flags: 0x44)), - ((uint)second.Length, Frame(second, second.Length, flags: 0x46)))); + byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool(((uint)first.Length, Synthetic.QuickLzFrame(first, first.Length, flags: 0x44)), + ((uint)second.Length, Synthetic.QuickLzFrame(second, second.Length, flags: 0x46)))); using var doc = PlaDocument.Open(new MemoryStream(bytes)); var pool = Assert.Single(doc.FindCompressedPools()); @@ -35,7 +32,7 @@ public void DecodesAnOverlappingQuickLzMatchAndLiteralTail() byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], "ABCD"u8.ToArray(), tokenBytes, "0123456789"u8.ToArray()); byte[] expected = "ABCDABCDABCDABCD0123456789"u8.ToArray(); using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( - Pool(((uint)expected.Length, Frame(body, expected.Length, flags: 0x45)))))); + Synthetic.QuickLzPool(((uint)expected.Length, Synthetic.QuickLzFrame(body, expected.Length, flags: 0x45)))))); Assert.Equal(expected, Assert.Single(doc.FindCompressedPools()).Decode()); } @@ -44,7 +41,7 @@ public void DecodesAnOverlappingQuickLzMatchAndLiteralTail() public void DecodedPoolCanBeLargerThanTheContainer() { byte[] body = Synthetic.Concat([0x10, 0, 0, 0x80], new byte[4], [0, 0, 190], new byte[11]); - byte[] bytes = Synthetic.Fdb(Pool((205, Frame(body, 205, flags: 0x47)))); + byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool((205, Synthetic.QuickLzFrame(body, 205, flags: 0x47)))); using var doc = PlaDocument.Open(new MemoryStream(bytes)); Assert.True(bytes.Length < 205); @@ -54,7 +51,7 @@ public void DecodedPoolCanBeLargerThanTheContainer() [Fact] public void MarkerInsideAnIndexedPayloadIsIgnored() { - byte[] pool = Pool((3, Frame([1, 2, 3], 3, flags: 0x44))); + byte[] pool = Synthetic.QuickLzPool((3, Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44))); using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( Synthetic.Block(PlaBlockMode.Raw, pool), pool))); @@ -66,7 +63,7 @@ public void MarkerInsideAnIndexedPayloadIsIgnored() [Fact] public void FindsMarkerAcrossTheScanWindowBoundary() { - byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Pool()); + byte[] bytes = Synthetic.Fdb(new byte[64 * 1024 - 8], Synthetic.QuickLzPool()); using var doc = PlaDocument.Open(new MemoryStream(bytes)); var pool = Assert.Single(doc.FindCompressedPools()); Assert.Equal(bytes.Length, pool.End); @@ -76,8 +73,8 @@ public void FindsMarkerAcrossTheScanWindowBoundary() [Fact] public void RejectsATruncatedFrameAtEndOfFile() { - byte[] frame = Frame([1, 2, 3], 3, flags: 0x44); - using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))[..^1]))); + byte[] frame = Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool((3, frame))[..^1]))); Assert.Empty(doc.FindCompressedPools()); } @@ -85,8 +82,8 @@ public void RejectsATruncatedFrameAtEndOfFile() [Fact] public void APoolWithInconsistentLengthsDoesNotHideAValidPool() { - byte[] frame = Frame([1, 2, 3], 3, flags: 0x44); - byte[] bytes = Synthetic.Fdb(Pool((4, frame)), Pool((3, frame))); + byte[] frame = Synthetic.QuickLzFrame([1, 2, 3], 3, flags: 0x44); + byte[] bytes = Synthetic.Fdb(Synthetic.QuickLzPool((4, frame)), Synthetic.QuickLzPool((3, frame))); using var doc = PlaDocument.Open(new MemoryStream(bytes)); var pool = Assert.Single(doc.FindCompressedPools()); @@ -97,8 +94,8 @@ public void APoolWithInconsistentLengthsDoesNotHideAValidPool() [Fact] public void DecodeRejectsAnUninitializedMatch() { - byte[] frame = Frame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45); - using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool((3, frame))))); + byte[] frame = Synthetic.QuickLzFrame([1, 0, 0, 0x80, 1, 0, 0, 0], 3, flags: 0x45); + using var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool((3, frame))))); var pool = Assert.Single(doc.FindCompressedPools()); Assert.Throws(pool.Decode); } @@ -106,7 +103,7 @@ public void DecodeRejectsAnUninitializedMatch() [Fact] public void DecodeAfterDisposeThrows() { - var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Pool()))); + var doc = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.QuickLzPool()))); var pool = Assert.Single(doc.FindCompressedPools()); doc.Dispose(); @@ -130,40 +127,4 @@ public void SavedFilePoolFramesDecodeToTheirDeclaredSize() if (decoded == 0) throw TestFiles.Missing("FDB fixture with QuickLZ pools"); } - private static byte[] Pool(params (uint Size, byte[] Frame)[] frames) - { - byte[] header = new byte[32]; - PoolMarker.CopyTo(header); - BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), (ulong)frames.Sum(f => (long)f.Size)); - BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(24), (ulong)frames.Length); - using var result = new MemoryStream(); - result.Write(header); - Span size = stackalloc byte[4]; - foreach (var (_, frame) in frames) - { - BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)frame.Length); - result.Write(size); - result.Write(frame); - } - return result.ToArray(); - } - - private static byte[] Frame(byte[] body, int size, byte flags) - { - int header = (flags & 2) == 0 ? 3 : 9; - byte[] frame = new byte[header + body.Length]; - frame[0] = flags; - if (header == 3) - { - frame[1] = checked((byte)frame.Length); - frame[2] = checked((byte)size); - } - else - { - BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(1), (uint)frame.Length); - BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(5), (uint)size); - } - body.CopyTo(frame, header); - return frame; - } } diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs new file mode 100644 index 0000000..0562b02 --- /dev/null +++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs @@ -0,0 +1,92 @@ +using Xunit; + +namespace PlaIO.Tests; + +public class PoolSubgroupTests +{ + [Fact] + public void RawSubgroupUsesThePayloadImmediatelyAfterTheSizes() + { + var guid = Guid.Parse("00112233-4455-6677-8899-aabbccddeeff"); + byte[] subgroup = Synthetic.Block(PlaBlockMode.Raw, "sample"u8.ToArray(), ver: 1, kind: 7, guid: guid); + subgroup[10] |= 4; // In a pool this flag adds no file-level extra field. + using var doc = OpenPool(subgroup); + + var record = Assert.Single(Assert.Single(doc.FindCompressedPools()).ReadSubgroups()); + Assert.Equal(0, record.Offset); + Assert.Equal(1, record.Version); + Assert.Equal(7, record.Kind); + Assert.Equal(guid, record.ObjectId); + Assert.Equal("sample"u8.ToArray(), record.Data); + Assert.False(record.Partial); + } + + [Theory] + [InlineData(1, false)] + [InlineData(3, true)] + public void ReportsTruncatedLiteralsSeparatelyFromValidBufferPadding(byte mask, bool partial) + { + byte[] subgroup = Synthetic.Block(PlaBlockMode.ZeroMask, [mask, 9], unpacked: 16); + using var doc = OpenPool(subgroup); + + var record = Assert.Single(Assert.Single(doc.FindCompressedPools()).ReadSubgroups()); + byte[] expected = new byte[16]; + expected[0] = 9; + Assert.Equal(expected, record.Data); + Assert.Equal(partial, record.Partial); + } + + [Fact] + public void LzSubgroupReportsPartialWithEmptyData() + { + byte[] subgroup = Synthetic.Block(PlaBlockMode.Lz, [1, 2, 3, 4], unpacked: 64); + using var doc = OpenPool(subgroup); + + var record = Assert.Single(Assert.Single(doc.FindCompressedPools()).ReadSubgroups()); + Assert.Equal(PlaBlockMode.Lz, record.Mode); + Assert.Empty(record.Data); + Assert.True(record.Partial); + } + + [Fact] + public void ShortHeaderCandidateDoesNotHideLaterRecord() + { + byte[] badHeader = [0x42, 0x4C, 0, 0, 0, 0, 1, 0, 1, 0xA0, 0x80]; + byte[] good = Synthetic.Block(PlaBlockMode.Raw, [42]); + using var doc = OpenPool(Synthetic.Concat(badHeader, good)); + + var record = Assert.Single(Assert.Single(doc.FindCompressedPools()).ReadSubgroups()); + Assert.Equal(badHeader.Length, record.Offset); + Assert.Equal(new byte[] { 42 }, record.Data); + } + + [Fact] + public void ReadSubgroupsAfterDisposeThrows() + { + var doc = OpenPool([]); + var pool = Assert.Single(doc.FindCompressedPools()); + doc.Dispose(); + + Assert.Throws(pool.ReadSubgroups); + } + + [SkippableFact] + public void ReadsSubgroupsFromSavedFiles() + { + using var corpus = new SampleCorpus(); + int pools = 0; + foreach (var sample in corpus.Fdb) + { + foreach (var pool in sample.Document.FindCompressedPools()) + { + if (pool.TotalDecompressed == 0) continue; + Assert.NotEmpty(pool.ReadSubgroups()); + pools++; + } + } + if (pools == 0) throw TestFiles.Missing("FDB fixture with nonempty QuickLZ pools"); + } + + private static PlaDocument OpenPool(byte[] data) => PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.QuickLzPool(((uint)data.Length, Synthetic.QuickLzFrame(data, data.Length, flags: 0x46)))))); +} diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index 75979fd..d037c8a 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -152,4 +152,44 @@ public static byte[] Png(params (string Type, byte[] Data)[] chunks) private static byte[] Le32(uint v) { var b = new byte[4]; BinaryPrimitives.WriteUInt32LittleEndian(b, v); return b; } private static byte[] Le64(ulong v) { var b = new byte[8]; BinaryPrimitives.WriteUInt64LittleEndian(b, v); return b; } + + private static ReadOnlySpan PoolMarker => + [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + public static byte[] QuickLzPool(params (uint Size, byte[] Frame)[] frames) + { + byte[] header = new byte[32]; + PoolMarker.CopyTo(header); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), (ulong)frames.Sum(f => (long)f.Size)); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(24), (ulong)frames.Length); + using var result = new MemoryStream(); + result.Write(header); + Span size = stackalloc byte[4]; + foreach (var (_, frame) in frames) + { + BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)frame.Length); + result.Write(size); + result.Write(frame); + } + return result.ToArray(); + } + + public static byte[] QuickLzFrame(byte[] body, int size, byte flags) + { + int header = (flags & 2) == 0 ? 3 : 9; + byte[] frame = new byte[header + body.Length]; + frame[0] = flags; + if (header == 3) + { + frame[1] = checked((byte)frame.Length); + frame[2] = checked((byte)size); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(1), (uint)frame.Length); + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(5), (uint)size); + } + body.CopyTo(frame, header); + return frame; + } }