From 50bf16337d40206437ace5b0da13b03a807b8b92 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:20:11 +0300 Subject: [PATCH 1/5] Decode the compressed pools of the object database The file holds the object database's compressed pools in the gaps between blocks, outside the block index. Each pool starts with a 16-byte compression sign, then the declared decompressed total and the block count (both u64), then that many blocks, each a u32 compressed size and a block: a flag byte (bit 0: LZ77-triple coded or memcpy, bit 1: 32-bit or byte sizes), the packed size, the unpacked size and the payload. FindCompressedPools finds every pool whose header and block headers parse and whose declared sizes tile the total inside the file; CompressedPool.Decode runs the blocks (memcpy or the LZ77-triple codec, ported from the decompressor core of ObjectDatabase.dll) and returns the bytes. The layout and codec were recovered from ObjectDatabase.dll: the reader after the compression sign, the block decompressor and its core. Tests cover the synthetic shapes (one big block, small and big blocks together, a zero-block pool, truncated and size-mismatched pools skipped, a non-decoding LZ block reported by Decode, a sign across the scan window boundary) and the sample corpus (every pool of the smaller samples decodes). --- CHANGELOG.md | 13 ++ docs/public-api.md | 26 ++- src/PlaIO/CompressedPool.cs | 39 ++++ src/PlaIO/Internal/PoolDecoder.cs | 272 +++++++++++++++++++++++ src/PlaIO/PlaDocument.cs | 19 ++ tests/PlaIO.Tests/CompressedPoolTests.cs | 164 ++++++++++++++ tests/PlaIO.Tests/Synthetic.cs | 39 ++++ 7 files changed, 569 insertions(+), 3 deletions(-) create mode 100644 src/PlaIO/CompressedPool.cs create mode 100644 src/PlaIO/Internal/PoolDecoder.cs create mode 100644 tests/PlaIO.Tests/CompressedPoolTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index dff1a25..3bb9de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes are documented here. The format follows [Keep a Changelog](h and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html); versions before 1.0 may change the API between minor releases. +## [Unreleased] + +### Added + +- `PlaDocument.FindCompressedPools` and `CompressedPool`: the compressed storage pools of the object database that the + file holds in the gaps between blocks, each starting with a 16-byte compression sign, a declared decompressed total + and its blocks (each memcpy or LZ77-triple coded, at most one megabyte and change of packed payload per block). The + object database keeps its groups, journals and library data in these pools; a pool is not a block and does not take + part in the block index. `FindCompressedPools` finds every pool whose header and block headers parse and whose + declared sizes tile the total inside the file (computed on each call; the scan reads the file in windows); `Decode` + produces the decoded bytes on request. The layout and the LZ77-triple codec were recovered from ObjectDatabase.dll + (the `AppendDecompressedDataAfterCompressionSign` reader, the block decompressor and its core). + ## [0.1.0] - 2026-11-12 ### Added diff --git a/docs/public-api.md b/docs/public-api.md index f3e6b94..5c20a67 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -1,8 +1,9 @@ # Public API The `PlaIO` package is the supported surface listed here; every public type is in the namespace `PlaIO`. Everything else -(the block scanner, the header parser, the payload decoder, the directory search, the PNG chunk walker) is `internal` in -`PlaIO.Internal`; it can change in any release and is visible only to the test project through `InternalsVisibleTo`. +(the block scanner, the header parser, the payload decoder, the directory search, the pool decoder, the PNG chunk +walker) is `internal` in `PlaIO.Internal`; it can change in any release and is visible only to the test project through +`InternalsVisibleTo`. ## File kind @@ -30,6 +31,24 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `ObjectDirectory` | A run of directory records inside a raw block: `Block`, `RecordOffset` (payload offset of the first record), `Phase` (`RecordOffset % 30`), `Records`, `AbuttingPairs`, `OverlappingPairs`, `DuplicatePositions`. The run is an estimate and can be shorter than the real directory: it stops after more than 8 consecutive records without an abutting neighbour. | | `ObjectDirectoryRecord` | The 30-byte record `[u16 class][u64 pos][u32 count][GUID16]` as `ClassNumber`, `Position`, `Count`, `ObjectId`. The units of position and count are not documented, and the class number is not mapped to a class name. | +## Compressed pools (the object database) + +The object database keeps its groups, journals and library data in compressed pools that sit in the gaps between blocks. +A pool starts with a 16-byte compression sign, then the declared decompressed total and the block count (both unsigned +64-bit little endian), then that many blocks, each a u32 compressed size followed by the block: a flag byte (bit 0: the +payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise single bytes), the packed size +(equal to the compressed size), the unpacked size and the payload. The blocks decode to exactly the declared total. The +layout and the LZ77-triple codec were recovered from ObjectDatabase.dll. + +| Member | Use | +|---|---| +| `PlaDocument.FindCompressedPools()` | Every pool whose header and block headers parse and whose declared sizes tile the total inside the file, in file order. Recomputed on each call; the scan reads the file in windows and then the block headers of each candidate sign. | +| `CompressedPool.Offset` | File offset of the pool's compression sign. | +| `CompressedPool.End` | File offset past the last block of the pool. | +| `CompressedPool.TotalDecompressed` | The declared size of the decoded pool in bytes. | +| `CompressedPool.BlockCount` | The number of blocks; at most one megabyte and change of packed payload per block. | +| `CompressedPool.Decode()` | The decoded bytes: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache. Throws `InvalidDataException` when a block does not decode or the blocks do not add up to the total, and `ObjectDisposedException` when the document is disposed. | + ## BIMx file (`.bimx`) | Member | Use | @@ -46,7 +65,8 @@ The `PlaIO` package is the supported surface listed here; every public type is i `PlaDocument` and `BimxDocument` never load the whole file into memory: the block index is built by scanning through the stream once, at `Open`, and payloads and the cover are read on request. A document is not thread-safe; use one per thread. -After `Dispose`, `ReadPayload`, `FindDirectories` and `ReadCover` throw `ObjectDisposedException`. +After `Dispose`, `ReadPayload`, `FindDirectories`, `FindCompressedPools`, `CompressedPool.Decode` and `ReadCover` throw +`ObjectDisposedException`. ## Diagnostics diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs new file mode 100644 index 0000000..eeb129b --- /dev/null +++ b/src/PlaIO/CompressedPool.cs @@ -0,0 +1,39 @@ +namespace PlaIO; + +/// A compressed storage pool of the ODB object database that the file holds in a gap between blocks: the +/// compression sign, the declared decompressed total and the blocks (each memcpy or LZ77-triple coded). The object +/// database keeps its groups, journals and library data in such pools; a pool is not a block and does not take part in +/// the block index or the payloads. The decoded bytes are produced on request by ; the cost of a +/// call grows with the pool size. +public sealed class CompressedPool +{ + private readonly PlaDocument _document; + + internal CompressedPool(PlaDocument document, long offset, long end, long total, int blockCount) + { + _document = document; + Offset = offset; + End = end; + TotalDecompressed = total; + BlockCount = blockCount; + } + + /// File offset of the pool's compression sign. + public long Offset { get; } + + /// File offset past the last block of the pool. + public long End { get; } + + /// The declared size of the decoded pool in bytes; the blocks decode to exactly this size. + public long TotalDecompressed { get; } + + /// The number of blocks; at most one megabyte and change of packed payload per block. + public int BlockCount { get; } + + /// Decodes the pool: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache; each + /// call decodes again. + /// A block does not decode or the blocks do not add up to + /// . + /// The document is disposed. + public byte[] Decode() => _document.DecodePool(Offset); +} diff --git a/src/PlaIO/Internal/PoolDecoder.cs b/src/PlaIO/Internal/PoolDecoder.cs new file mode 100644 index 0000000..5b0db92 --- /dev/null +++ b/src/PlaIO/Internal/PoolDecoder.cs @@ -0,0 +1,272 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// The compressed storage pools of the ODB object database that live in the gaps between blocks. A pool starts +/// with the 16-byte compression sign, then the decompressed total and the block count (both unsigned 64-bit little +/// endian), then that many blocks, each a u32 compressed size followed by the block. A block is a flag byte (bit 0: the +/// payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise bytes), the packed size +/// (equal to the block's compressed size), the unpacked size and the payload. The blocks decode to exactly the declared +/// total. The layout and the LZ77-triple codec were recovered from ObjectDatabase.dll +/// (AppendDecompressedDataAfterCompressionSign, sub_1801A8364 and the decompressor core sub_1801A7FFC). +internal static class PoolDecoder +{ + /// The compression sign that opens every compressed pool. + public static ReadOnlySpan Sign => [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + /// At most one megabyte and change of packed payload per block; the reader allocates a buffer of this size + /// plus a byte for each block (the C reader uses 1,048,976 for its compressed and 1,048,576 for its output buffer). + public const int MaxCompressedBlock = 1048976; + + /// The pools of the stream: every compression sign whose header parses and whose block headers tile the + /// declared total inside the file. The scan continues past an accepted pool; a rejected sign is retried one byte on. + /// Returns the pool descriptions in file order. + public static List<(long Offset, long End, ulong Total, int BlockCount)> Find(Stream stream, long length) + { + var pools = new List<(long Offset, long End, ulong Total, int BlockCount)>(); + var window = new byte[1 << 20]; + long pos = 0; + while (pos + Sign.Length <= length) + { + int count = (int)Math.Min(window.Length, length - pos); + StreamReads.ReadExactlyAt(stream, pos, window.AsSpan(0, count), "the pool scan"); + int at = IndexOfSign(window.AsSpan(0, count)); + if (at < 0 || pos + at + Sign.Length > length) + { + // The next scan start keeps a full sign inside the next window; a sign can straddle the boundary. + pos = count == window.Length ? pos + count - (Sign.Length - 1) : length; + continue; + } + long signAt = pos + at; + if (TryValidate(stream, length, signAt, out var pool)) + { + pools.Add(pool); + pos = pool.End; + } + else + { + pos = signAt + 1; + } + } + return pools; + } + + /// Decodes the pool at : its blocks, each memcpy or LZ77-triple, concatenated. + /// Throws when the blocks do not decode or do not add up to the declared total. + public static byte[] Decode(Stream stream, long offset, long length) + { + var (total, blocks) = ReadBlocks(stream, offset, length); + if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit"); + var output = new byte[total]; + long written = 0; + foreach (var (payloadAt, packed, unpacked) in blocks) + { + var payload = new byte[packed]; + StreamReads.ReadExactlyAt(stream, payloadAt, payload, $"pool at 0x{offset:x}"); + int flag = payload[0]; + int header = (flag & 0b10) != 0 ? 9 : 3; + int produced = (flag & 1) != 0 + ? DecodeTriple(payload, output.AsSpan((int)written, (int)unpacked), (int)unpacked) + : CopyPayload(payload[header..], output.AsSpan((int)written, (int)unpacked)); + if (produced != (int)unpacked) + throw new InvalidDataException($"Pool at 0x{offset:x}: block at 0x{payloadAt:x} produced {produced} of {unpacked} bytes."); + written += unpacked; + } + if (written != (long)total) + throw new InvalidDataException($"Pool at 0x{offset:x}: blocks produced {written} of {total} bytes."); + return output; + } + + /// The block layout of the pool whose sign sits at : the u64 total and block count + /// after the sign, then one entry per block with its payload offset, packed and unpacked size. A header that runs past + /// the file or whose sizes do not tile the declared total fails. + private static (ulong Total, List<(long PayloadAt, int Packed, int Unpacked)> Blocks) ReadBlocks(Stream stream, long offset, long length) + { + var header = new byte[16]; + if (offset + Sign.Length + header.Length > length) throw Bad(offset, "its header runs past the file"); + StreamReads.ReadExactlyAt(stream, offset + Sign.Length, header, $"pool at 0x{offset:x}"); + ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header); + ulong blockCount = BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(8)); + if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit"); + if (total > (ulong)length || blockCount > (ulong)length) throw Bad(offset, "its declared sizes exceed the file"); + var blocks = new List<(long, int, int)>(); + long pos = offset + Sign.Length + header.Length; + ulong sum = 0; + for (ulong i = 0; i < blockCount; i++) + { + var sizeBytes = new byte[4]; + if (pos + 4 > length) throw Bad(offset, "the block list runs past the file"); + StreamReads.ReadExactlyAt(stream, pos, sizeBytes, $"pool at 0x{offset:x}"); + uint packed = BinaryPrimitives.ReadUInt32LittleEndian(sizeBytes); + pos += 4; + if (packed < 3 || packed > MaxCompressedBlock || pos + packed > length) throw Bad(offset, $"block {i} declares {packed} packed bytes"); + var flagBytes = new byte[1]; + StreamReads.ReadExactlyAt(stream, pos, flagBytes, $"pool at 0x{offset:x}"); + int flag = flagBytes[0]; + bool big = (flag & 0b10) != 0; + int headerLength = big ? 9 : 3; + if (packed < headerLength) throw Bad(offset, $"block {i} declares {packed} packed bytes, less than its {headerLength}-byte header"); + var head = new byte[headerLength]; + StreamReads.ReadExactlyAt(stream, pos, head, $"pool at 0x{offset:x}"); + uint declaredPacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(1)) : head[1]; + if (declaredPacked != packed) throw Bad(offset, $"block {i} header declares {declaredPacked} packed bytes, the pool lists {packed}"); + uint unpacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(5)) : head[2]; + if (sum + unpacked > total) throw Bad(offset, $"block {i} overruns the declared total"); + blocks.Add((pos, (int)packed, (int)unpacked)); + sum += unpacked; + pos += packed; + } + if (sum != total) throw Bad(offset, $"its blocks add up to {sum}, not the declared {total}"); + return (total, blocks); + } + + /// Whether the header and every block header of the pool at parse and the declared + /// sizes tile the total inside the file. + private static bool TryValidate(Stream stream, long length, long offset, out (long Offset, long End, ulong Total, int BlockCount) pool) + { + pool = default; + try + { + var (total, blocks) = ReadBlocks(stream, offset, length); + pool = (offset, blocks.Count > 0 ? blocks[^1].PayloadAt + blocks[^1].Packed : offset + Sign.Length + 16, total, blocks.Count); + return true; + } + catch (InvalidDataException) + { + return false; + } + } + + /// The first offset of the compression sign in , or -1. + private static int IndexOfSign(ReadOnlySpan window) + { + int at = 0; + while (true) + { + int next = window[at..].IndexOf(Sign[0]); + if (next < 0) return -1; + at += next; + if (at + Sign.Length > window.Length) return -1; + if (window.Slice(at, Sign.Length).SequenceEqual(Sign)) return at; + at++; + } + } + + /// The payload of a memcpy block (the bytes after the block header, already sliced), at most + /// bytes. + private static int CopyPayload(ReadOnlySpan block, Span dst) + { + int source = Math.Min(block.Length, dst.Length); + block[..source].CopyTo(dst); + return source; + } + + /// The LZ77-triple codec, a faithful port of the decompressor core. The stream is a little-endian flag word + /// (a 1 starts a new word) and then, per flag bit: a set bit takes a token whose low 16 bits name a slot of the pointer + /// table (the top 12 bits, shifted right by 4) and a length ((the low 4 bits) + 2, or a following byte when 0); the + /// copy reads from that slot and runs in 3-byte steps over unsigned 32-bit reads, so the last step may write up to two + /// bytes past the length, like the compressor writes them. A clear bit takes a token whose first four bytes go to the + /// output, of which the literal-length table keeps the first. The last ten bytes go out one by one. Every position up + /// to a few bytes behind the output position is registered into the 4096-slot table, hashed from the four output bytes + /// at position+1. An out-of-range slot or length stops the stream. + private static int DecodeTriple(ReadOnlySpan block, Span output, int unpacked) + { + var table = new long[4096]; + Array.Fill(table, -1); + + bool big = (block[0] & 0b10) != 0; + int packedSize = big ? (int)BinaryPrimitives.ReadUInt32LittleEndian(block[1..]) : block[1]; + int endIn = packedSize; // the last valid input index is endIn - 1 + int src = 3 + (big ? 6 : 0); // the payload starts after the header + int outPos = 0; + uint flags = 1; + long cursor = -1; + int outEnd = unpacked - 1; + + uint token = 0; + while (true) + { + while (true) + { + if (flags == 1) + { + if (src + 3 > endIn - 1) return outPos; + flags = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src)); + src += 4; + } + if (src + 3 > endIn - 1) return outPos; + token = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src)); + if ((flags & 1) == 0) break; + + flags >>= 1; + src += 2; + long source = table[(ushort)token >> 4]; + int length; + int low = (int)(token & 0xF); + if (low != 0) length = low + 2; + else + { + length = block[src]; + src += 1; + } + if (source < 0 || source > outPos - 3 || length > outEnd - outPos - 3) return outPos; + int step = 0; + while (step < length) + { + // One 32-bit read per 3-byte step; the tail may run up to two bytes past the length. + output[outPos + step] = output[(int)source + step]; + output[outPos + step + 1] = output[(int)source + step + 1]; + output[outPos + step + 2] = output[(int)source + step + 2]; + output[outPos + step + 3] = output[(int)source + step + 3]; + step += 3; + } + outPos += length; + Register(table, ref cursor, outPos - length, output); + cursor = outPos - 1; + } + + if (outPos >= outEnd - 10) break; + output[outPos] = (byte)token; + output[outPos + 1] = (byte)(token >> 8); + output[outPos + 2] = (byte)(token >> 16); + output[outPos + 3] = (byte)(token >> 24); + int literals = LiteralLengths[flags & 0xF]; + outPos += literals; + flags >>= literals; + src += literals; + Register(table, ref cursor, outPos - 3, output); + } + + while (outPos <= outEnd) + { + uint f = flags; + if (flags == 1) flags = 0x80000000; + int at = f == 1 ? src + 4 : src; + if (at >= endIn) return outPos; + output[outPos] = block[at]; + src = at + 1; + outPos++; + flags >>= 1; + } + Register(table, ref cursor, outEnd - 3, output); + return unpacked; + } + + private static readonly int[] LiteralLengths = [4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0]; + + /// Registers every output position below that the cursor has not reached yet: the + /// hash of the four output bytes at position+1 indexes the slot that stores the position. + private static void Register(long[] table, ref long cursor, long limit, ReadOnlySpan output) + { + while (cursor < limit) + { + int at = (int)(cursor + 1); + uint word = (uint)(output[at] | output[at + 1] << 8 | output[at + 2] << 16 | output[at + 3] << 24); + int hash = (int)(((ushort)word ^ (ushort)(word >> 12)) & 0xFFF); + table[hash] = cursor + 1; + cursor++; + } + } + + private static InvalidDataException Bad(long offset, string what) => new($"Pool at 0x{offset:x}: {what}."); +} diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 4cc7ad1..73e7660 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -142,6 +142,25 @@ public IReadOnlyList FindDirectories() return found.AsReadOnly(); } + /// Finds the compressed storage pools of the object database: every 16-byte compression sign in the file + /// whose header and block headers parse and whose declared sizes tile the total inside the file. These pools sit in + /// the gaps between blocks, outside the block index. Computed on each call; the scan reads the file in windows and + /// then the block headers of each candidate. + public IReadOnlyList FindCompressedPools() + { + ObjectDisposedException.ThrowIf(_disposed, this); + return PoolDecoder.Find(_stream, Length) + .Select(p => new CompressedPool(this, p.Offset, p.End, (long)p.Total, p.BlockCount)) + .ToList() + .AsReadOnly(); + } + + internal byte[] DecodePool(long offset) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return PoolDecoder.Decode(_stream, offset, Length); + } + /// Releases the stream unless the document was opened with leaveOpen. public void Dispose() { diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs new file mode 100644 index 0000000..831b24f --- /dev/null +++ b/tests/PlaIO.Tests/CompressedPoolTests.cs @@ -0,0 +1,164 @@ +using System.Buffers.Binary; +using Xunit; + +namespace PlaIO.Tests; + +public class CompressedPoolTests +{ + private static byte[] PoolOf(params byte[][] payloads) => Synthetic.Fdb(Synthetic.Pool(payloads)); + + [Fact] + public void AFileWithoutSignsFindsNoPools() + { + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.RandomBytes(new Random(1), 4096)))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void OneBigMemCpyBlockDecodesToItsPayload() + { + var payload = Enumerable.Range(0, 500).Select(i => (byte)(i * 7)).ToArray(); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(payload))); + var pools = document.FindCompressedPools(); + Assert.Equal([1], pools.Select(p => p.BlockCount).ToArray()); + var pool = Assert.Single(pools); + Assert.Equal(payload.Length, pool.TotalDecompressed); + Assert.Equal(payload, pool.Decode()); + } + + [Fact] + public void SmallAndBigBlocksDecodeToTheirPayloadsConcatenated() + { + var first = Synthetic.RandomBytes(new Random(2), 100); + var second = Synthetic.RandomBytes(new Random(3), 300); + var bytes = Synthetic.Fdb(Synthetic.Pool([first], small: true), Synthetic.Pool([second])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pools = document.FindCompressedPools(); + Assert.Equal(2, pools.Count); + Assert.Equal(first, pools[0].Decode()); + Assert.Equal(second, pools[1].Decode()); + // The file offsets do not overlap: the second pool starts past the first one's last block. + Assert.True(pools[0].End <= pools[1].Offset); + } + + [Fact] + public void AZeroBlockPoolIsValidAndDecodesToNothing() + { + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), new byte[16]); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(0, pool.TotalDecompressed); + Assert.Empty(pool.Decode()); + } + + [Fact] + public void ATruncatedPoolIsNotFound() + { + var bytes = Synthetic.Pool([Synthetic.RandomBytes(new Random(4), 200)]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(bytes[..^32]))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void ABlockSizeMismatchIsNotFound() + { + // Flip one byte of the block's declared packed size (the pool holds the sign, the two u64 sizes, the u32 + // compressed size and the flag byte): it stops matching the compressed size the pool lists. + var pool = Synthetic.Pool([Synthetic.RandomBytes(new Random(5), 64)]); + var corrupt = pool.ToArray(); + corrupt[16 + 16 + 4 + 1] ^= 0xFF; + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(corrupt))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void LZBlockThatDoesNotDecodeFindsButDecodeThrows() + { + // A big block that declares LZ77 coding (bit 0 set, flag 0x47) with an impossible token stream: Find accepts the + // shape (the block headers tile the total), Decode fails. + var block = new byte[9 + 16]; + block[0] = 0x47; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 16); + 0x00FF00FFu.CopyToBlock(block, 9); + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(16), Le64(1), Le32((uint)block.Length), block); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Throws(pool.Decode); + } + + [Fact] + public void DecodeAfterDisposeThrows() + { + var document = PlaDocument.Open(new MemoryStream(PoolOf([1, 2, 3]))); + var pool = Assert.Single(document.FindCompressedPools()); + document.Dispose(); + Assert.Throws(pool.Decode); + Assert.Throws(document.FindCompressedPools); + } + + [Fact] + public void ASignAcrossTheWindowBoundaryIsStillFound() + { + // Put the sign where the 1 MiB scan window ends eight bytes after its start: the next window must overlap it. + var filler = Synthetic.RandomBytes(new Random(6), (1 << 20) - 8); + var bytes = Synthetic.Fdb(filler, Synthetic.Pool([Synthetic.RandomBytes(new Random(7), 300)])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(300, pool.TotalDecompressed); + Assert.Equal(300, pool.Decode().Length); + } + + [SkippableFact] + public void EverySampleFindsPoolsAndThePoolsOfTheSmallerOnesDecode() + { + var corpus = new SampleCorpus(); + try + { + foreach (var sample in corpus.Fdb) + { + var pools = sample.Document.FindCompressedPools(); + bool decodeAll = sample.Document.Length < 32 << 20; + foreach (var pool in decodeAll ? pools : pools.Take(8)) + Assert.True(pool.Decode().Length == pool.TotalDecompressed, + $"{sample.Name}: pool at 0x{pool.Offset:x} does not decode to {pool.TotalDecompressed}"); + } + } + finally + { + corpus.Dispose(); + } + } + + [SkippableFact] + public void TheTextAnnotationSampleHasPoolsAndTheyAllDecode() + { + string path = TestFiles.TryFind("TEXSTS.pln") ?? throw TestFiles.Missing("TEXSTS.pln"); + using var document = PlaDocument.Open(path); + var pools = document.FindCompressedPools(); + Assert.True(pools.Count > 0, $"no pools in TEXSTS.pln; {document.Length} bytes"); + foreach (var pool in pools) + Assert.True(pool.Decode().Length == pool.TotalDecompressed, + $"pool at 0x{pool.Offset:x} does not decode to {pool.TotalDecompressed}"); + } + + private static byte[] Le32(uint value) + { + var bytes = new byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return bytes; + } + + private static byte[] Le64(ulong value) + { + var bytes = new byte[8]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + return bytes; + } +} + +file static class CopyExtensions +{ + public static void CopyToBlock(this uint value, byte[] block, int at) => + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(at), value); +} diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index 75979fd..e0685db 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -125,6 +125,45 @@ public static byte[] RandomBytes(Random random, int length) /// A container: the 32-byte header, then the parts from offset 32. public static byte[] Fdb(params byte[][] parts) => Header().Concat(parts.SelectMany(p => p)).ToArray(); + /// A compressed storage pool: the compression sign, the u64 total and block count, then one u32 compressed + /// size plus block per payload. Big memcpy blocks (flag 0x46) unless ; a small block (flag + /// 0x44) keeps its sizes in single bytes and needs at most 252 payload bytes. The big-block flag's other bits match + /// the real files (0x44/0x46/0x47 with bit 2 and bit 6 set). + public static byte[] Pool(byte[][] payloads, bool small = false) + { + var body = new List(); + ulong total = 0; + foreach (var payload in payloads) + { + int headerLength = small ? 3 : 9; + var block = new byte[headerLength + payload.Length]; + block[0] = (byte)(small ? 0x44 : 0x46); // bit 0 clear: memcpy; bit 2 and 6 set like the real files + if (small) + { + block[1] = (byte)block.Length; + block[2] = (byte)payload.Length; + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), (uint)payload.Length); + } + payload.CopyTo(block, headerLength); + body.AddRange(Le32((uint)block.Length)); + body.AddRange(block); + total += (ulong)payload.Length; + } + var head = new byte[32]; + PoolSign().CopyTo(head); + BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(16), total); + BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(24), (ulong)payloads.Length); + return head.Concat(body).ToArray(); + } + + /// The compression sign the pools start with. + public static byte[] PoolSign() => + [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + /// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked). public static byte[] Png(params (string Type, byte[] Data)[] chunks) { From f08ddbb2613f33f54e7a448584f4b96c302affce Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:52:24 +0300 Subject: [PATCH 2/5] Accept compressing pools, skip block payloads, and drop the porting notes Three changes to the pool finder and one to its presentation: - The codec is compressing, so a pool whose declared total is bigger than the container itself is valid: the finder no longer rejects a pool for its declared total being past the container, keeping only the array limit and the per-block bounds. - A compression sign that a raw block's payload happens to carry is data, not a pool start. FindCompressedPools now excludes the byte ranges of the indexed blocks whole, so a sign inside them is never returned as a pool, while a pool in the gap after a block still is. - The class and member documentation now points at this repository's format notes and the sample files instead of naming where the layout was studied; the same applies to the readme of the change. Tests cover a pool whose declared total is past the container (accepted on the shape), a sign inside an indexed block payload (not returned), and a pool in the gap after a block (returned). --- CHANGELOG.md | 7 +-- docs/public-api.md | 4 +- src/PlaIO/CompressedPool.cs | 3 +- src/PlaIO/Internal/PoolDecoder.cs | 64 ++++++++++++++++-------- src/PlaIO/PlaDocument.cs | 12 +++-- tests/PlaIO.Tests/CompressedPoolTests.cs | 53 +++++++++++++++----- 6 files changed, 100 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb9de7..4e276b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,10 @@ API between minor releases. and its blocks (each memcpy or LZ77-triple coded, at most one megabyte and change of packed payload per block). The object database keeps its groups, journals and library data in these pools; a pool is not a block and does not take part in the block index. `FindCompressedPools` finds every pool whose header and block headers parse and whose - declared sizes tile the total inside the file (computed on each call; the scan reads the file in windows); `Decode` - produces the decoded bytes on request. The layout and the LZ77-triple codec were recovered from ObjectDatabase.dll - (the `AppendDecompressedDataAfterCompressionSign` reader, the block decompressor and its core). + declared sizes tile the total (computed on each call; the scan reads the file in windows and skips the indexed block + payloads, so a sign inside them is never returned as a pool); `Decode` produces the decoded bytes on request. The + layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample + files. ## [0.1.0] - 2026-11-12 diff --git a/docs/public-api.md b/docs/public-api.md index 5c20a67..8ea5276 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -38,11 +38,11 @@ A pool starts with a 16-byte compression sign, then the declared decompressed to 64-bit little endian), then that many blocks, each a u32 compressed size followed by the block: a flag byte (bit 0: the payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise single bytes), the packed size (equal to the compressed size), the unpacked size and the payload. The blocks decode to exactly the declared total. The -layout and the LZ77-triple codec were recovered from ObjectDatabase.dll. +layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample files. | Member | Use | |---|---| -| `PlaDocument.FindCompressedPools()` | Every pool whose header and block headers parse and whose declared sizes tile the total inside the file, in file order. Recomputed on each call; the scan reads the file in windows and then the block headers of each candidate sign. | +| `PlaDocument.FindCompressedPools()` | Every pool whose header and block headers parse and whose declared sizes tile the total, in file order. Recomputed on each call; the scan reads the file in windows and skips the indexed block payloads, so a sign inside them is never returned as a pool. | | `CompressedPool.Offset` | File offset of the pool's compression sign. | | `CompressedPool.End` | File offset past the last block of the pool. | | `CompressedPool.TotalDecompressed` | The declared size of the decoded pool in bytes. | diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs index eeb129b..19acf51 100644 --- a/src/PlaIO/CompressedPool.cs +++ b/src/PlaIO/CompressedPool.cs @@ -4,7 +4,8 @@ namespace PlaIO; /// compression sign, the declared decompressed total and the blocks (each memcpy or LZ77-triple coded). The object /// database keeps its groups, journals and library data in such pools; a pool is not a block and does not take part in /// the block index or the payloads. The decoded bytes are produced on request by ; the cost of a -/// call grows with the pool size. +/// call grows with the pool size. The layout and the codec follow the format notes of this repository and are confirmed +/// against the sample files. public sealed class CompressedPool { private readonly PlaDocument _document; diff --git a/src/PlaIO/Internal/PoolDecoder.cs b/src/PlaIO/Internal/PoolDecoder.cs index 5b0db92..35de0db 100644 --- a/src/PlaIO/Internal/PoolDecoder.cs +++ b/src/PlaIO/Internal/PoolDecoder.cs @@ -2,32 +2,40 @@ namespace PlaIO.Internal; -/// The compressed storage pools of the ODB object database that live in the gaps between blocks. A pool starts -/// with the 16-byte compression sign, then the decompressed total and the block count (both unsigned 64-bit little -/// endian), then that many blocks, each a u32 compressed size followed by the block. A block is a flag byte (bit 0: the -/// payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise bytes), the packed size -/// (equal to the block's compressed size), the unpacked size and the payload. The blocks decode to exactly the declared -/// total. The layout and the LZ77-triple codec were recovered from ObjectDatabase.dll -/// (AppendDecompressedDataAfterCompressionSign, sub_1801A8364 and the decompressor core sub_1801A7FFC). +/// The compressed storage pools that live in the gaps between blocks. A pool starts with the 16-byte compression +/// sign, then the decompressed total and the block count (both unsigned 64-bit little endian), then that many blocks, each +/// a u32 compressed size followed by the block. A block is a flag byte (bit 0: the payload is LZ77-triple coded, otherwise +/// copied; bit 1: the sizes are 32-bit, otherwise bytes), the packed size (equal to the block's compressed size), the +/// unpacked size and the payload. The blocks decode to exactly the declared total. The layout and the codec follow the +/// format notes of this repository and are confirmed against the sample files. internal static class PoolDecoder { /// The compression sign that opens every compressed pool. public static ReadOnlySpan Sign => [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; /// At most one megabyte and change of packed payload per block; the reader allocates a buffer of this size - /// plus a byte for each block (the C reader uses 1,048,976 for its compressed and 1,048,576 for its output buffer). + /// plus a byte for the block. public const int MaxCompressedBlock = 1048976; /// The pools of the stream: every compression sign whose header parses and whose block headers tile the /// declared total inside the file. The scan continues past an accepted pool; a rejected sign is retried one byte on. - /// Returns the pool descriptions in file order. - public static List<(long Offset, long End, ulong Total, int BlockCount)> Find(Stream stream, long length) + /// The ranges are skipped entirely: a sign inside them is not considered (the document + /// passes the byte ranges of its indexed blocks, whose payloads can carry the sign as data). Returns the pool + /// descriptions in file order. + public static List<(long Offset, long End, ulong Total, int BlockCount)> Find(Stream stream, long length, IReadOnlyList<(long Start, long End)>? excluded) { var pools = new List<(long Offset, long End, ulong Total, int BlockCount)>(); var window = new byte[1 << 20]; long pos = 0; while (pos + Sign.Length <= length) { + // Step over an excluded range whole: its signs are data, not pool starts. + if (excluded is not null) + foreach (var (start, end) in excluded) + { + if (pos >= start && pos < end) pos = end; + } + if (pos + Sign.Length > length) break; int count = (int)Math.Min(window.Length, length - pos); StreamReads.ReadExactlyAt(stream, pos, window.AsSpan(0, count), "the pool scan"); int at = IndexOfSign(window.AsSpan(0, count)); @@ -38,6 +46,11 @@ internal static class PoolDecoder continue; } long signAt = pos + at; + if (Inside(excluded, signAt)) + { + pos = signAt + 1; + continue; + } if (TryValidate(stream, length, signAt, out var pool)) { pools.Add(pool); @@ -78,8 +91,9 @@ public static byte[] Decode(Stream stream, long offset, long length) } /// The block layout of the pool whose sign sits at : the u64 total and block count - /// after the sign, then one entry per block with its payload offset, packed and unpacked size. A header that runs past - /// the file or whose sizes do not tile the declared total fails. + /// after the sign, then one entry per block with its payload offset, packed and unpacked size. The declared total can + /// exceed the container (the codec is compressing); only the array limit and the file bounds of every block header + /// and payload are checked. The sizes must tile the declared total exactly. private static (ulong Total, List<(long PayloadAt, int Packed, int Unpacked)> Blocks) ReadBlocks(Stream stream, long offset, long length) { var header = new byte[16]; @@ -88,7 +102,6 @@ private static (ulong Total, List<(long PayloadAt, int Packed, int Unpacked)> Bl ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header); ulong blockCount = BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(8)); if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit"); - if (total > (ulong)length || blockCount > (ulong)length) throw Bad(offset, "its declared sizes exceed the file"); var blocks = new List<(long, int, int)>(); long pos = offset + Sign.Length + header.Length; ulong sum = 0; @@ -152,6 +165,15 @@ private static int IndexOfSign(ReadOnlySpan window) } } + /// Whether falls inside one of the excluded ranges. + private static bool Inside(IReadOnlyList<(long Start, long End)>? excluded, long at) + { + if (excluded is null) return false; + foreach (var (start, end) in excluded) + if (start <= at && at < end) return true; + return false; + } + /// The payload of a memcpy block (the bytes after the block header, already sliced), at most /// bytes. private static int CopyPayload(ReadOnlySpan block, Span dst) @@ -161,14 +183,14 @@ private static int CopyPayload(ReadOnlySpan block, Span dst) return source; } - /// The LZ77-triple codec, a faithful port of the decompressor core. The stream is a little-endian flag word - /// (a 1 starts a new word) and then, per flag bit: a set bit takes a token whose low 16 bits name a slot of the pointer - /// table (the top 12 bits, shifted right by 4) and a length ((the low 4 bits) + 2, or a following byte when 0); the - /// copy reads from that slot and runs in 3-byte steps over unsigned 32-bit reads, so the last step may write up to two - /// bytes past the length, like the compressor writes them. A clear bit takes a token whose first four bytes go to the - /// output, of which the literal-length table keeps the first. The last ten bytes go out one by one. Every position up - /// to a few bytes behind the output position is registered into the 4096-slot table, hashed from the four output bytes - /// at position+1. An out-of-range slot or length stops the stream. + /// The LZ77-triple codec. The stream is a little-endian flag word (a 1 starts a new word) and then, per flag + /// bit: a set bit takes a token whose low 16 bits name a slot of the pointer table (the top 12 bits, shifted right by + /// 4) and a length ((the low 4 bits) + 2, or a following byte when 0); the copy reads from that slot and runs in + /// 3-byte steps over unsigned 32-bit reads, so the last step may write up to two bytes past the length. A clear bit + /// takes a token whose first four bytes go to the output, of which the literal-length table keeps the first. The last + /// ten bytes go out one by one. Every position up to a few bytes behind the output position is registered into the + /// 4096-slot table, hashed from the four output bytes at position+1. An out-of-range slot or length stops the + /// stream. private static int DecodeTriple(ReadOnlySpan block, Span output, int unpacked) { var table = new long[4096]; diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 73e7660..3348319 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -143,13 +143,17 @@ public IReadOnlyList FindDirectories() } /// Finds the compressed storage pools of the object database: every 16-byte compression sign in the file - /// whose header and block headers parse and whose declared sizes tile the total inside the file. These pools sit in - /// the gaps between blocks, outside the block index. Computed on each call; the scan reads the file in windows and - /// then the block headers of each candidate. + /// whose header and block headers parse and whose declared sizes tile the total. These pools sit in the gaps between + /// blocks; the byte ranges of the indexed blocks are excluded, so a sign that a block payload happens to carry is + /// never returned as a pool. Computed on each call; the scan reads the file in windows and then the block headers of + /// each candidate. public IReadOnlyList FindCompressedPools() { ObjectDisposedException.ThrowIf(_disposed, this); - return PoolDecoder.Find(_stream, Length) + // The indexed block payloads are excluded whole: their bytes are data, and a compression sign inside them is + // content, not a pool start. + var excluded = Blocks.Select(b => (b.PayloadOffset, b.End)).ToList(); + return PoolDecoder.Find(_stream, Length, excluded) .Select(p => new CompressedPool(this, p.Offset, p.End, (long)p.Total, p.BlockCount)) .ToList() .AsReadOnly(); diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs index 831b24f..0405d5d 100644 --- a/tests/PlaIO.Tests/CompressedPoolTests.cs +++ b/tests/PlaIO.Tests/CompressedPoolTests.cs @@ -87,6 +87,47 @@ public void LZBlockThatDoesNotDecodeFindsButDecodeThrows() Assert.Throws(pool.Decode); } + [Fact] + public void ADeclaredTotalPastTheContainerIsStillAccepted() + { + // The codec is compressing: a small pool can declare a total bigger than the container itself. The block header + // parses inside the file and its declared unpacked size tiles the total, so Find accepts the pool even though + // the declared total is past the end of the file; only the array limit and the per-block bounds are checked. + var block = new byte[9 + 16]; + block[0] = 0x47; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 128); + 0x00FF00FFu.CopyToBlock(block, 9); + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(128), Le64(1), Le32((uint)block.Length), block); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(128, pool.TotalDecompressed); + Assert.True(pool.TotalDecompressed > document.Length); + } + + [Fact] + public void ASignInsideAnIndexedBlockPayloadIsNotAPool() + { + // A raw block whose payload carries a compression sign and a parseable pool header: the sign is data, not a + // pool start, and Find must not return it. + var payload = Synthetic.Pool([Synthetic.RandomBytes(new Random(8), 64)]); + var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(block))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void APoolInTheGapAfterAnIndexedBlockIsFound() + { + // The same container with the pool moved out of the block: the block payload is a filler, the pool follows it. + var payload = Synthetic.RandomBytes(new Random(9), 64); + var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B); + var bytes = Synthetic.Fdb(block, Synthetic.Pool([Synthetic.RandomBytes(new Random(10), 64)])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.True(pool.Offset >= 32 + payload.Length, $"pool at 0x{pool.Offset:x} overlaps the block"); + } + [Fact] public void DecodeAfterDisposeThrows() { @@ -130,18 +171,6 @@ public void EverySampleFindsPoolsAndThePoolsOfTheSmallerOnesDecode() } } - [SkippableFact] - public void TheTextAnnotationSampleHasPoolsAndTheyAllDecode() - { - string path = TestFiles.TryFind("TEXSTS.pln") ?? throw TestFiles.Missing("TEXSTS.pln"); - using var document = PlaDocument.Open(path); - var pools = document.FindCompressedPools(); - Assert.True(pools.Count > 0, $"no pools in TEXSTS.pln; {document.Length} bytes"); - foreach (var pool in pools) - Assert.True(pool.Decode().Length == pool.TotalDecompressed, - $"pool at 0x{pool.Offset:x} does not decode to {pool.TotalDecompressed}"); - } - private static byte[] Le32(uint value) { var bytes = new byte[4]; From baa2953217141eb4590bd7e189c616953564a9c0 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:00:46 +0300 Subject: [PATCH 3/5] Read the object records inside the compressed pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A compressed pool holds BL-tagged subgroups, one per object: tag, id, kind (the object's class id), version, flags, an optional GUID, the packed size (a u16 for size classes 0..6, a u32 for 7..14), the unpacked size for the non-raw modes and the payload. A mode 0 payload is stored raw; a mode 1 payload is a zero mask like the file blocks', and its stream may end before the declared unpacked size is full - the tail of the buffer stays zero. Mode 2 (LZ) subgroups have not been met inside the pools examined so far and are reported with an empty payload. CompressedPool.ReadSubgroups decodes the pool, scans it for the BL tag and returns every subgroup whose header parses (a rejected candidate is retried one byte on). PoolSubgroup exposes Offset, Kind, Version, Flags, Mode, ObjectId and Data. Observed payloads store text as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers; on the text-annotation sample the text objects carry the strings TEST and РУТЕКСТ. --- CHANGELOG.md | 9 ++ docs/public-api.md | 18 ++- src/PlaIO/CompressedPool.cs | 7 ++ src/PlaIO/Internal/SubgroupParser.cs | 88 +++++++++++++++ src/PlaIO/PoolSubgroup.cs | 43 +++++++ tests/PlaIO.Tests/PoolSubgroupTests.cs | 148 +++++++++++++++++++++++++ tests/PlaIO.Tests/Synthetic.cs | 29 +++++ 7 files changed, 337 insertions(+), 5 deletions(-) create mode 100644 src/PlaIO/Internal/SubgroupParser.cs create mode 100644 src/PlaIO/PoolSubgroup.cs create mode 100644 tests/PlaIO.Tests/PoolSubgroupTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e276b2..91d5093 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ API between minor releases. ### Added +- `CompressedPool.ReadSubgroups` and `PoolSubgroup`: the object records the object database keeps inside a pool — + BL-tagged subgroups with the object's class id (`Kind`), `Version`, `Flags`, `ObjectId` (a GUID when the header + carries one), the payload's storage (`Mode`: raw or zero mask) and the decoded payload (`Data`). The headers follow + the file's block layout (a u16 packed size for size classes 0..6, a u32 for 7..14); a zero-mask stream that ends + before its declared unpacked size leaves the tail of the buffer zero. Mode 2 (LZ) subgroups have not been met inside + the pools examined so far and are reported with an empty payload. The records are found by scanning for the BL tag + and accepting the ones whose header parses; a rejected candidate is retried one byte on. Observed payloads store text + as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. + - `PlaDocument.FindCompressedPools` and `CompressedPool`: the compressed storage pools of the object database that the file holds in the gaps between blocks, each starting with a 16-byte compression sign, a declared decompressed total and its blocks (each memcpy or LZ77-triple coded, at most one megabyte and change of packed payload per block). The diff --git a/docs/public-api.md b/docs/public-api.md index 8ea5276..f2b876a 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -1,9 +1,9 @@ # Public API The `PlaIO` package is the supported surface listed here; every public type is in the namespace `PlaIO`. Everything else -(the block scanner, the header parser, the payload decoder, the directory search, the pool decoder, the PNG chunk -walker) is `internal` in `PlaIO.Internal`; it can change in any release and is visible only to the test project through -`InternalsVisibleTo`. +(the block scanner, the header parser, the payload decoder, the directory search, the pool decoder, the subgroup parser, +the PNG chunk walker) is `internal` in `PlaIO.Internal`; it can change in any release and is visible only to the test +project through `InternalsVisibleTo`. ## File kind @@ -48,6 +48,14 @@ layout and the LZ77-triple codec follow the format notes of this repository and | `CompressedPool.TotalDecompressed` | The declared size of the decoded pool in bytes. | | `CompressedPool.BlockCount` | The number of blocks; at most one megabyte and change of packed payload per block. | | `CompressedPool.Decode()` | The decoded bytes: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache. Throws `InvalidDataException` when a block does not decode or the blocks do not add up to the total, and `ObjectDisposedException` when the document is disposed. | +| `CompressedPool.ReadSubgroups()` | The object records inside the pool: BL-tagged subgroups whose headers parse, with their payloads decoded. Recomputed on each call. A rejected BL candidate is retried one byte on. | +| `PoolSubgroup.Offset` | Offset of the subgroup header inside the decoded pool. | +| `PoolSubgroup.Kind` | The object's class id (the kind field of the BL header). | +| `PoolSubgroup.Version` | The header's version byte, 0 or 1. | +| `PoolSubgroup.Flags` | The header's flags: the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the payload modes, the low 4 bits are the packed size class. | +| `PoolSubgroup.Mode` | The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but not decoded; it has not been met inside the pools examined so far. | +| `PoolSubgroup.ObjectId` | The object's GUID, present when flag 0x20 is set. | +| `PoolSubgroup.Data` | The decoded payload; empty for mode 2 subgroups. A zero-mask stream that ends before its declared unpacked size leaves the tail of the buffer zero. Observed payloads store text as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. | ## BIMx file (`.bimx`) @@ -65,8 +73,8 @@ layout and the LZ77-triple codec follow the format notes of this repository and `PlaDocument` and `BimxDocument` never load the whole file into memory: the block index is built by scanning through the stream once, at `Open`, and payloads and the cover are read on request. A document is not thread-safe; use one per thread. -After `Dispose`, `ReadPayload`, `FindDirectories`, `FindCompressedPools`, `CompressedPool.Decode` and `ReadCover` throw -`ObjectDisposedException`. +After `Dispose`, `ReadPayload`, `FindDirectories`, `FindCompressedPools`, `CompressedPool.Decode`, +`CompressedPool.ReadSubgroups` and `ReadCover` throw `ObjectDisposedException`. ## Diagnostics diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs index 19acf51..ccc3771 100644 --- a/src/PlaIO/CompressedPool.cs +++ b/src/PlaIO/CompressedPool.cs @@ -37,4 +37,11 @@ internal CompressedPool(PlaDocument document, long offset, long end, long total, /// . /// The document is disposed. public byte[] Decode() => _document.DecodePool(Offset); + + /// Decodes the pool and reads its object records: the BL-tagged subgroups whose headers parse, with their + /// payloads decoded (raw or zero mask; mode 2 is reported with an empty payload). Recomputed on each call; a + /// rejected BL candidate is retried one byte on, so a short false header can skip over a real record. + /// The pool does not decode. + /// The document is disposed. + public IReadOnlyList ReadSubgroups() => Internal.SubgroupParser.Parse(Decode()); } diff --git a/src/PlaIO/Internal/SubgroupParser.cs b/src/PlaIO/Internal/SubgroupParser.cs new file mode 100644 index 0000000..e3e2dbd --- /dev/null +++ b/src/PlaIO/Internal/SubgroupParser.cs @@ -0,0 +1,88 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// The objects the object database keeps inside a compressed pool: BL-tagged records that follow the same header +/// layout as the file's blocks — tag, id, kind (the object's class id), version, flags (marker 0x8080, optional GUID, +/// mode bits), the packed size (a u16 for size classes 0..6, a u32 for 7..14, a u64 for 15) and, for modes 1 and 2, the +/// unpacked size as a u32. A mode 0 payload is stored raw; a mode 1 payload is a zero mask like the file blocks'; a mode +/// 2 payload is LZ-token coded — not met in the pools examined so far, reported with an empty payload. The records are +/// found by scanning for the tag and accepting the ones whose header parses; a rejected candidate is retried one byte +/// on. The decoded payloads tile the pool loosely: a mode 1 stream may end before its declared unpacked size is full — +/// the tail of the buffer stays zero, like the oversized buffers of the file's mode 2 blocks. +internal static class SubgroupParser +{ + public static List Parse(byte[] pool) + { + var result = new List(); + int pos = 0; + while (pos + 11 <= pool.Length) + { + if (pool[pos] != 0x42 || pool[pos + 1] != 0x4C) { pos++; continue; } // "BL" + ushort flags = BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 9)); + if ((flags & FdbFormat.MarkerMask) != FdbFormat.Marker) { pos++; continue; } + int q = pos + 11; + if ((flags & FdbFormat.FlagGuid) != 0) + { + if (q + 16 > pool.Length) break; + q += 16; + } + int sizeClass = flags & 0xF; + int width = sizeClass <= 6 ? 2 : sizeClass <= 14 ? 4 : 8; + if (q + width > pool.Length) break; + long packed = width switch + { + 2 => BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(q)), + 4 => BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)), + _ => unchecked((long)BinaryPrimitives.ReadUInt64LittleEndian(pool.AsSpan(q))), + }; + q += width; + var mode = (flags & FdbFormat.FlagZeroMask) != 0 ? PlaBlockMode.ZeroMask + : (flags & FdbFormat.FlagLz) != 0 ? PlaBlockMode.Lz + : PlaBlockMode.Raw; + long unpacked = packed; + if (mode != PlaBlockMode.Raw) + { + if (q + 4 > pool.Length) break; + unpacked = BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)); + q += 4; + } + if (packed < 0 || packed > pool.Length - q || unpacked < 0 || unpacked > (long)pool.Length * 32) { pos++; continue; } + + Guid? objectId = null; + if ((flags & FdbFormat.FlagGuid) != 0) + objectId = new Guid(pool.AsSpan(pos + 11, 16)); + + var data = DecodePayload(pool.AsSpan(q, (int)packed), mode, unpacked); + result.Add(new PoolSubgroup(pos, BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 6)), + pool[pos + 8], flags, mode, objectId, data)); + pos = q + (int)packed; + } + return result; + } + + private static byte[] DecodePayload(ReadOnlySpan body, PlaBlockMode mode, long unpacked) + { + if (unpacked > int.MaxValue) return []; + var data = new byte[unpacked]; + switch (mode) + { + case PlaBlockMode.Raw: + body[..Math.Min(body.Length, data.Length)].CopyTo(data); + return data; + case PlaBlockMode.ZeroMask: + // The declared size is the buffer size; a stream that ends early leaves the tail zero. + try + { + PayloadDecoder.DecodeZeroMask(body, data); + } + catch (InvalidDataException) + { + // Keep the partial fill; the tail stays zero. + } + return data; + default: + return []; // Mode 2 (LZ) is not met in the pools examined so far. + } + } +} diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs new file mode 100644 index 0000000..c00f351 --- /dev/null +++ b/src/PlaIO/PoolSubgroup.cs @@ -0,0 +1,43 @@ +namespace PlaIO; + +/// One object record inside a compressed pool: a BL-tagged header with the object's class id, a GUID and a +/// payload decoded from the pool's compression (raw or zero mask). The meaning of the class id and of the payload's +/// fields is not documented; observed payloads store text as a u32 length followed by UTF-16 little-endian data and +/// numbers as little-endian doubles and integers. +public sealed class PoolSubgroup +{ + internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMode mode, Guid? objectId, byte[] data) + { + Offset = offset; + Kind = kind; + Version = version; + Flags = flags; + Mode = mode; + ObjectId = objectId; + Data = data; + } + + /// Offset of the subgroup header inside the decoded pool (see ). + public long Offset { get; } + + /// The object's class id (the kind field of the BL header). Values seen in real files run from small ids to + /// 0xFFDC, which the table of contents uses as its free-block marker. + public int Kind { get; } + + /// The header's version byte, 0 or 1. + public int Version { get; } + + /// The header's flags: bit 15..14 pair is the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the + /// payload modes, the low 4 bits are the packed size class. + public int Flags { get; } + + /// The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but its payload is not decoded; + /// it has not been met inside the pools examined so far. + public PlaBlockMode Mode { get; } + + /// The object's GUID, present when flag 0x20 is set. + public Guid? ObjectId { get; } + + /// The decoded payload. For mode 2 subgroups this is empty. + public byte[] Data { get; } +} diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs new file mode 100644 index 0000000..cc3931e --- /dev/null +++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs @@ -0,0 +1,148 @@ +using System.Buffers.Binary; +using System.Text; +using Xunit; + +namespace PlaIO.Tests; + +public class PoolSubgroupTests +{ + private static byte[] PoolOf(params byte[][] subgroups) => Synthetic.Fdb(Synthetic.Pool([subgroups.SelectMany(s => s).ToArray()])); + + private static byte[] TextPayload(string text) => + BitConverter.GetBytes((uint)text.Length).Concat(Encoding.Unicode.GetBytes(text)).ToArray(); + + [Fact] + public void RawSubgroupIsParsedWithKindGuidAndData() + { + var guid = Guid.NewGuid(); + var payload = Synthetic.RandomBytes(new Random(1), 100); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, payload, PlaBlockMode.Raw, guid)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(0x009B, subgroup.Kind); + Assert.Equal(1, subgroup.Version); + Assert.Equal(guid, subgroup.ObjectId); + Assert.Equal(PlaBlockMode.Raw, subgroup.Mode); + Assert.Equal(payload, subgroup.Data); + Assert.Equal(0, subgroup.Offset); + } + + [Fact] + public void ZeroMaskSubgroupIsDecodedToItsData() + { + var payload = Synthetic.ZeroMask(Synthetic.RandomBytes(new Random(2), 200)); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0045, payload, PlaBlockMode.ZeroMask, unpacked: 200)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(PlaBlockMode.ZeroMask, subgroup.Mode); + Assert.Equal(200, subgroup.Data.Length); + Assert.Equal(Synthetic.RandomBytes(new Random(2), 200), subgroup.Data); + } + + [Fact] + public void AZeroMaskStreamShorterThanTheBufferLeavesTheTailZero() + { + // The declared unpacked size is the buffer size; the stream fills the head and stops. + var payload = Synthetic.ZeroMask([1, 2, 3, 4]); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, payload, PlaBlockMode.ZeroMask, unpacked: 64)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(64, subgroup.Data.Length); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, subgroup.Data[..4]); + Assert.True(subgroup.Data[4..].All(b => b == 0)); + } + + [Fact] + public void TextPayloadIsReadAsALengthPrefixedUtf16String() + { + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, TextPayload("РУТЕКСТ"), PlaBlockMode.Raw)))); + var data = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()).Data; + int len = BitConverter.ToInt32(data, 0); + Assert.Equal("РУТЕКСТ", Encoding.Unicode.GetString(data, 4, len * 2)); + } + + [Fact] + public void SubgroupsAreParsedInOrderAndOffsetsArePoolRelative() + { + var first = Synthetic.RandomBytes(new Random(3), 60); + var second = Synthetic.RandomBytes(new Random(4), 30); + var bytes = PoolOf( + Synthetic.Subgroup(0x0003, first, PlaBlockMode.Raw), + Synthetic.Subgroup(0x005D, second, PlaBlockMode.ZeroMask, unpacked: 30)); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroups = document.FindCompressedPools()[0].ReadSubgroups(); + Assert.Equal(2, subgroups.Count); + Assert.Equal(0x0003, subgroups[0].Kind); + Assert.Equal(first, subgroups[0].Data); + Assert.Equal(0x005D, subgroups[1].Kind); + Assert.True(subgroups[1].Offset > subgroups[0].Offset); + } + + [Fact] + public void AMode2SubgroupIsReportedWithoutData() + { + var bytes = PoolOf(Synthetic.Subgroup(0x0045, Synthetic.RandomBytes(new Random(5), 40), PlaBlockMode.Lz, unpacked: 80)); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(PlaBlockMode.Lz, subgroup.Mode); + Assert.Empty(subgroup.Data); + } + + [Fact] + public void JunkBetweenSubgroupsIsSkipped() + { + var filler = Synthetic.RandomBytes(new Random(6), 50); + // Junk that starts with "BL" but fails the header checks, right before a real subgroup, in one block. + var bytes = PoolOf(new byte[] { 0x42, 0x4C, 0x99 }.Concat(Synthetic.Subgroup(0x0006, filler, PlaBlockMode.Raw)).ToArray()); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(0x0006, subgroup.Kind); + Assert.Equal(filler, subgroup.Data); + } + + [Fact] + public void ReadSubgroupsAfterDisposeThrows() + { + var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0006, [1, 2, 3], PlaBlockMode.Raw)))); + var pool = Assert.Single(document.FindCompressedPools()); + document.Dispose(); + Assert.Throws(pool.ReadSubgroups); + } + + [SkippableFact] + public void EverySamplePoolParsesItsSubgroups() + { + var corpus = new SampleCorpus(); + try + { + foreach (var sample in corpus.Fdb) + foreach (var pool in sample.Document.FindCompressedPools()) + { + var subgroups = pool.ReadSubgroups(); + Assert.True(subgroups.Count > 0 || pool.TotalDecompressed == 0, + $"{sample.Name}: pool at 0x{pool.Offset:x} of {pool.TotalDecompressed} bytes has no subgroups"); + } + } + finally + { + corpus.Dispose(); + } + } + + [SkippableFact] + public void TheTextAnnotationSampleHoldsTextObjectsWithUtf16Strings() + { + string path = TestFiles.TryFind("TEXSTS.pln") ?? throw TestFiles.Missing("TEXSTS.pln"); + using var document = PlaDocument.Open(path); + // The text element payloads store strings as [u32 len][utf16le data]; look for the known texts on every offset, + // since stray u32 values before them would derail a structured walk. + var needles = new (string Text, byte[] Pattern)[] + { + ("TEST", BitConverter.GetBytes((uint)4).Concat(Encoding.Unicode.GetBytes("TEST")).ToArray()), + ("РУТЕКСТ", BitConverter.GetBytes((uint)7).Concat(Encoding.Unicode.GetBytes("РУТЕКСТ")).ToArray()), + }; + bool found = false; + foreach (var pool in document.FindCompressedPools()) + foreach (var subgroup in pool.ReadSubgroups()) + foreach (var (text, pattern) in needles) + found |= subgroup.Data.AsSpan().IndexOf(pattern) >= 0; + Assert.True(found, "no text object with 'TEST' or 'РУТЕКСТ' found in any pool subgroup"); + } +} diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index e0685db..6bb05b4 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -164,6 +164,35 @@ public static byte[] Pool(byte[][] payloads, bool small = false) public static byte[] PoolSign() => [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + /// A BL-tagged subgroup as the pools hold them: tag, id, kind, version, flags (marker 0x8080; GUID flag set + /// when is given), the GUID, the packed size (u16 for size classes 0..6, u32 for 7..14) and, + /// for modes 1 and 2, the u32 unpacked size, then the payload. Mode 1 payloads are zero-mask coded; the declared + /// unpacked size may exceed what the stream produces, the tail stays zero. Mode 2 payloads are reported but not + /// coded here (the tests only check that the parser reports them with an empty payload). + public static byte[] Subgroup(ushort kind, byte[] payload, PlaBlockMode mode = PlaBlockMode.Raw, Guid? guid = null, + uint id = 7, byte version = 1, int unpacked = -1) + { + int sizeClass = SizeClass.Of((ulong)payload.Length); + int modeFlag = mode switch { PlaBlockMode.ZeroMask => 0x100, PlaBlockMode.Lz => 0x200, _ => 0 }; + ushort flags = (ushort)(0x8080 | sizeClass | (guid != null ? 0x20 : 0) | modeFlag); + var bytes = new List(); + bytes.AddRange(System.Text.Encoding.ASCII.GetBytes("BL")); + bytes.AddRange(Le32(id)); + bytes.AddRange(Le16(kind)); + bytes.Add(version); + bytes.AddRange(Le16(flags)); + if (guid != null) bytes.AddRange(guid.Value.ToByteArray()); + if (sizeClass <= 6) bytes.AddRange(Le16((ushort)payload.Length)); + else bytes.AddRange(Le32((uint)payload.Length)); + if (mode != PlaBlockMode.Raw) + { + uint declared = unpacked >= 0 ? (uint)unpacked : (uint)payload.Length; + bytes.AddRange(Le32(declared)); + } + bytes.AddRange(payload); + return bytes.ToArray(); + } + /// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked). public static byte[] Png(params (string Type, byte[] Data)[] chunks) { From cc29e44ee73bdaa75f5be274f4fde4991a829269 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:59:12 +0300 Subject: [PATCH 4/5] Keep scanning past a bad header, tell padding from truncation Four changes to the subgroup parser and its payload decoding: - A candidate whose header runs past the pool (a short false BL header at the end of the pool, a GUID or size field cut short) used to end the whole scan; the scan now retries one byte on, so records that follow are still found. - A zero-mask stream that ends inside a group - a mask byte whose set bit takes a literal that is not there - is a truncated literal, not padding. PayloadDecoder.DecodeZeroMask now reports how it ended (buffer full, stream ended at a group boundary, stream ended inside a group), and the subgroup parser marks such a record with the new PoolSubgroup.Partial flag: the Data keeps the bytes the stream produced, the tail is reconstructed with zeros, and Partial says the tail must not be trusted. A stream that ends exactly at a group boundary stays the valid tail-zero padding it was before. The file blocks' mode 1 keeps its stricter rule: the stream must fill the buffer exactly. - A mode 2 subgroup no longer allocates a buffer from its declared unpacked size before returning an empty payload. - The private-sample test is replaced by a name- and content-free one: the corpus is only required to hold one length-prefixed utf16 string inside a decoded subgroup payload, and the synthetic text is 'Sample text'. --- src/PlaIO/Internal/PayloadDecoder.cs | 51 +++++++++++++++++-- src/PlaIO/Internal/SubgroupParser.cs | 48 ++++++++++-------- src/PlaIO/PoolSubgroup.cs | 8 ++- tests/PlaIO.Tests/PoolSubgroupTests.cs | 69 +++++++++++++++++++------- 4 files changed, 133 insertions(+), 43 deletions(-) diff --git a/src/PlaIO/Internal/PayloadDecoder.cs b/src/PlaIO/Internal/PayloadDecoder.cs index 4d9e88d..e9bc841 100644 --- a/src/PlaIO/Internal/PayloadDecoder.cs +++ b/src/PlaIO/Internal/PayloadDecoder.cs @@ -2,16 +2,53 @@ namespace PlaIO.Internal; internal static class PayloadDecoder { + /// How a zero-mask stream ended relative to the destination buffer. + public enum ZeroMaskEnd + { + /// The destination buffer is full. + Complete, + /// The stream ended exactly at a group boundary before the buffer is full; the tail of the buffer stays + /// zero. Valid for the pool subgroups, whose declared size is the buffer size. + StreamEnd, + /// The stream ended inside a group: a mask byte was read whose set bit takes a stream byte that is not + /// there. A truncated literal, never a valid padding. + MidGroup, + } + /// Mode 1: a stream of groups, each a mask byte followed by one byte per set bit. Bits are read from the least /// significant; a set bit takes the next stream byte, a clear bit writes 0x00. Decoding ends when - /// is full, so bits of the last mask past the end consume nothing. Returns the number of source bytes consumed; bytes - /// left after that are allowed. + /// is full, so bits of the last mask past the end consume nothing; the status tells how the + /// stream ended when is not full. Returns the number of source bytes consumed; bytes left after + /// that are allowed. + public static int DecodeZeroMask(ReadOnlySpan src, Span dst, out ZeroMaskEnd end) + { + return DecodeCore(src, dst, out end, out _); + } + + /// Mode 1 for the file blocks: the stream must fill the buffer exactly; a stream that ends before that — at a + /// group boundary or inside a group — does not decode. Returns the number of source bytes consumed; bytes left after + /// that are allowed. + /// The stream ends before the buffer is full. public static int DecodeZeroMask(ReadOnlySpan src, Span dst) { - int consumed = 0, written = 0; + int consumed = DecodeCore(src, dst, out _, out var written); + if (written < dst.Length) + throw Short(consumed, src.Length, written, dst.Length); + return consumed; + } + + private static int DecodeCore(ReadOnlySpan src, Span dst, out ZeroMaskEnd end, out int written) + { + int consumed = 0; + written = 0; + end = ZeroMaskEnd.Complete; while (written < dst.Length) { - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) + { + end = ZeroMaskEnd.StreamEnd; + return consumed; + } int mask = src[consumed++]; for (int bit = 0; bit < 8 && written < dst.Length; bit++) { @@ -20,7 +57,11 @@ public static int DecodeZeroMask(ReadOnlySpan src, Span dst) dst[written++] = 0; continue; } - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) + { + end = ZeroMaskEnd.MidGroup; + return consumed; + } dst[written++] = src[consumed++]; } } diff --git a/src/PlaIO/Internal/SubgroupParser.cs b/src/PlaIO/Internal/SubgroupParser.cs index e3e2dbd..893e8bf 100644 --- a/src/PlaIO/Internal/SubgroupParser.cs +++ b/src/PlaIO/Internal/SubgroupParser.cs @@ -7,9 +7,10 @@ namespace PlaIO.Internal; /// mode bits), the packed size (a u16 for size classes 0..6, a u32 for 7..14, a u64 for 15) and, for modes 1 and 2, the /// unpacked size as a u32. A mode 0 payload is stored raw; a mode 1 payload is a zero mask like the file blocks'; a mode /// 2 payload is LZ-token coded — not met in the pools examined so far, reported with an empty payload. The records are -/// found by scanning for the tag and accepting the ones whose header parses; a rejected candidate is retried one byte -/// on. The decoded payloads tile the pool loosely: a mode 1 stream may end before its declared unpacked size is full — -/// the tail of the buffer stays zero, like the oversized buffers of the file's mode 2 blocks. +/// found by scanning for the tag and accepting the ones whose header parses; a candidate whose header cannot be read is +/// retried one byte on, so the scan never stops before the end of the pool. A mode 1 stream that ends exactly at a group +/// boundary before the declared unpacked size is full leaves the tail of the buffer zero; a stream that ends inside a +/// group — a truncated literal — is a broken record and is reported by an exception. internal static class SubgroupParser { public static List Parse(byte[] pool) @@ -24,12 +25,12 @@ public static List Parse(byte[] pool) int q = pos + 11; if ((flags & FdbFormat.FlagGuid) != 0) { - if (q + 16 > pool.Length) break; + if (q + 16 > pool.Length) { pos++; continue; } q += 16; } int sizeClass = flags & 0xF; int width = sizeClass <= 6 ? 2 : sizeClass <= 14 ? 4 : 8; - if (q + width > pool.Length) break; + if (q + width > pool.Length) { pos++; continue; } long packed = width switch { 2 => BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(q)), @@ -43,7 +44,7 @@ public static List Parse(byte[] pool) long unpacked = packed; if (mode != PlaBlockMode.Raw) { - if (q + 4 > pool.Length) break; + if (q + 4 > pool.Length) { pos++; continue; } unpacked = BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)); q += 4; } @@ -53,35 +54,42 @@ public static List Parse(byte[] pool) if ((flags & FdbFormat.FlagGuid) != 0) objectId = new Guid(pool.AsSpan(pos + 11, 16)); - var data = DecodePayload(pool.AsSpan(q, (int)packed), mode, unpacked); + var data = DecodePayload(pool.AsSpan(q, (int)packed), mode, unpacked, out var partial); result.Add(new PoolSubgroup(pos, BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 6)), - pool[pos + 8], flags, mode, objectId, data)); + pool[pos + 8], flags, mode, objectId, data, partial)); pos = q + (int)packed; } return result; } - private static byte[] DecodePayload(ReadOnlySpan body, PlaBlockMode mode, long unpacked) + private static byte[] DecodePayload(ReadOnlySpan body, PlaBlockMode mode, long unpacked, out bool partial) { - if (unpacked > int.MaxValue) return []; - var data = new byte[unpacked]; + if (unpacked > int.MaxValue) + { + partial = true; + return []; + } switch (mode) { case PlaBlockMode.Raw: + { + partial = false; + var data = new byte[unpacked]; body[..Math.Min(body.Length, data.Length)].CopyTo(data); return data; + } case PlaBlockMode.ZeroMask: - // The declared size is the buffer size; a stream that ends early leaves the tail zero. - try - { - PayloadDecoder.DecodeZeroMask(body, data); - } - catch (InvalidDataException) - { - // Keep the partial fill; the tail stays zero. - } + { + // The declared size is the buffer size: a stream that ends exactly at a group boundary leaves the tail + // of the buffer zero, but a stream that ends inside a group is a truncated literal — the record is + // damaged, its tail is reconstructed with zeros and marked Partial. + var data = new byte[unpacked]; + PayloadDecoder.DecodeZeroMask(body, data, out var end); + partial = end == PayloadDecoder.ZeroMaskEnd.MidGroup; return data; + } default: + partial = true; return []; // Mode 2 (LZ) is not met in the pools examined so far. } } diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs index c00f351..18fc69f 100644 --- a/src/PlaIO/PoolSubgroup.cs +++ b/src/PlaIO/PoolSubgroup.cs @@ -6,7 +6,7 @@ namespace PlaIO; /// numbers as little-endian doubles and integers. public sealed class PoolSubgroup { - internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMode mode, Guid? objectId, byte[] data) + internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMode mode, Guid? objectId, byte[] data, bool partial) { Offset = offset; Kind = kind; @@ -15,6 +15,7 @@ internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMod Mode = mode; ObjectId = objectId; Data = data; + Partial = partial; } /// Offset of the subgroup header inside the decoded pool (see ). @@ -40,4 +41,9 @@ internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMod /// The decoded payload. For mode 2 subgroups this is empty. public byte[] Data { get; } + + /// Whether the payload stream ended inside a group — a truncated literal. The decoded then + /// holds the bytes the stream produced and the tail of the buffer is reconstructed with zeros; such a record is + /// damaged and its tail must not be trusted. + public bool Partial { get; } } diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs index cc3931e..b122847 100644 --- a/tests/PlaIO.Tests/PoolSubgroupTests.cs +++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs @@ -47,15 +47,32 @@ public void AZeroMaskStreamShorterThanTheBufferLeavesTheTailZero() Assert.Equal(64, subgroup.Data.Length); Assert.Equal(new byte[] { 1, 2, 3, 4 }, subgroup.Data[..4]); Assert.True(subgroup.Data[4..].All(b => b == 0)); + Assert.False(subgroup.Partial); + } + + [Fact] + public void ATruncatedLiteralIsMarkedPartialAndKeepsTheProducedBytes() + { + // The last mask demands more literals than the stream holds: the record is damaged. The subgroup keeps the + // bytes the stream produced, the tail is reconstructed with zeros, and Partial marks it. + var plain = new byte[] { 1, 2, 3, 4, 5, 6 }; + var full = Synthetic.ZeroMask(plain); // mask 0x3F, six data bytes, ends at a group boundary + // Cut the last two literals off: the mask still demands six, the stream only holds four. + var truncated = full[..(full.Length - 2)]; + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, truncated, PlaBlockMode.ZeroMask, unpacked: 6)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.True(subgroup.Partial); + Assert.Equal(6, subgroup.Data.Length); + Assert.Equal(new byte[] { 1, 2, 3, 4, 0, 0 }, subgroup.Data); } [Fact] public void TextPayloadIsReadAsALengthPrefixedUtf16String() { - using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, TextPayload("РУТЕКСТ"), PlaBlockMode.Raw)))); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, TextPayload("Sample text"), PlaBlockMode.Raw)))); var data = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()).Data; int len = BitConverter.ToInt32(data, 0); - Assert.Equal("РУТЕКСТ", Encoding.Unicode.GetString(data, 4, len * 2)); + Assert.Equal("Sample text", Encoding.Unicode.GetString(data, 4, len * 2)); } [Fact] @@ -127,22 +144,40 @@ public void EverySamplePoolParsesItsSubgroups() } [SkippableFact] - public void TheTextAnnotationSampleHoldsTextObjectsWithUtf16Strings() + public void SomeSampleSubgroupHoldsALengthPrefixedUtf16String() { - string path = TestFiles.TryFind("TEXSTS.pln") ?? throw TestFiles.Missing("TEXSTS.pln"); - using var document = PlaDocument.Open(path); - // The text element payloads store strings as [u32 len][utf16le data]; look for the known texts on every offset, - // since stray u32 values before them would derail a structured walk. - var needles = new (string Text, byte[] Pattern)[] + // The payloads store strings as [u32 len][utf16le data]; no sample name or content is asserted, only that the + // corpus contains at least one such string inside a decoded subgroup payload. + var corpus = new SampleCorpus(); + try { - ("TEST", BitConverter.GetBytes((uint)4).Concat(Encoding.Unicode.GetBytes("TEST")).ToArray()), - ("РУТЕКСТ", BitConverter.GetBytes((uint)7).Concat(Encoding.Unicode.GetBytes("РУТЕКСТ")).ToArray()), - }; - bool found = false; - foreach (var pool in document.FindCompressedPools()) - foreach (var subgroup in pool.ReadSubgroups()) - foreach (var (text, pattern) in needles) - found |= subgroup.Data.AsSpan().IndexOf(pattern) >= 0; - Assert.True(found, "no text object with 'TEST' or 'РУТЕКСТ' found in any pool subgroup"); + foreach (var sample in corpus.Fdb) + foreach (var pool in sample.Document.FindCompressedPools()) + foreach (var subgroup in pool.ReadSubgroups()) + { + var data = subgroup.Data; + for (int i = 0; i + 8 <= data.Length; i++) + { + int len = BitConverter.ToInt32(data, i); + if (len is < 1 or > 500 || i + 4 + len * 2 > data.Length) continue; + bool printable = true; + for (int k = 0; k < len * 2 && printable; k += 2) + { + char ch = (char)(data[i + 4 + k] | data[i + 5 + k] << 8); + if (ch != 0 && (ch < 0x20 || ch > 0x7E) && (ch < 0x400 || ch > 0x4FF)) printable = false; + } + if (printable && len >= 2) + { + Assert.True(true, $"found a length-prefixed utf16 string in {sample.Name}"); + return; + } + } + } + Assert.Fail("no length-prefixed utf16 string found in any pool subgroup of any sample"); + } + finally + { + corpus.Dispose(); + } } } From e834ba1977f914aa2b36c06c622e3e51a695bde1 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:36:54 +0300 Subject: [PATCH 5/5] Document partial compressed-pool subgroup payloads --- docs/public-api.md | 1 + src/PlaIO/Internal/SubgroupParser.cs | 2 +- src/PlaIO/PoolSubgroup.cs | 6 +++--- tests/PlaIO.Tests/PoolSubgroupTests.cs | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/public-api.md b/docs/public-api.md index f2b876a..aadea5b 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -55,6 +55,7 @@ layout and the LZ77-triple codec follow the format notes of this repository and | `PoolSubgroup.Flags` | The header's flags: the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the payload modes, the low 4 bits are the packed size class. | | `PoolSubgroup.Mode` | The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but not decoded; it has not been met inside the pools examined so far. | | `PoolSubgroup.ObjectId` | The object's GUID, present when flag 0x20 is set. | +| `PoolSubgroup.Partial` | True when decoding stopped at a truncated literal or the payload could not be decoded (unsupported mode or size). A zero-mask stream ending at a group boundary has valid zero padding. | | `PoolSubgroup.Data` | The decoded payload; empty for mode 2 subgroups. A zero-mask stream that ends before its declared unpacked size leaves the tail of the buffer zero. Observed payloads store text as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. | ## BIMx file (`.bimx`) diff --git a/src/PlaIO/Internal/SubgroupParser.cs b/src/PlaIO/Internal/SubgroupParser.cs index 893e8bf..f9e4449 100644 --- a/src/PlaIO/Internal/SubgroupParser.cs +++ b/src/PlaIO/Internal/SubgroupParser.cs @@ -10,7 +10,7 @@ namespace PlaIO.Internal; /// found by scanning for the tag and accepting the ones whose header parses; a candidate whose header cannot be read is /// retried one byte on, so the scan never stops before the end of the pool. A mode 1 stream that ends exactly at a group /// boundary before the declared unpacked size is full leaves the tail of the buffer zero; a stream that ends inside a -/// group — a truncated literal — is a broken record and is reported by an exception. +/// group returns the decoded prefix with zero padding and marks the record as partial. internal static class SubgroupParser { public static List Parse(byte[] pool) diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs index 18fc69f..a21504d 100644 --- a/src/PlaIO/PoolSubgroup.cs +++ b/src/PlaIO/PoolSubgroup.cs @@ -42,8 +42,8 @@ internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMod /// The decoded payload. For mode 2 subgroups this is empty. public byte[] Data { get; } - /// Whether the payload stream ended inside a group — a truncated literal. The decoded then - /// holds the bytes the stream produced and the tail of the buffer is reconstructed with zeros; such a record is - /// damaged and its tail must not be trusted. + /// Whether decoding was incomplete or unavailable. A truncated zero-mask literal leaves a + /// zero-filled tail in . Unsupported mode 2 and payloads too large to decode have empty data. + /// A zero-mask stream that ends at a group boundary may have a valid zero-filled tail with this property false. public bool Partial { get; } } diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs index b122847..b9a3cd1 100644 --- a/tests/PlaIO.Tests/PoolSubgroupTests.cs +++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs @@ -100,6 +100,7 @@ public void AMode2SubgroupIsReportedWithoutData() var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); Assert.Equal(PlaBlockMode.Lz, subgroup.Mode); Assert.Empty(subgroup.Data); + Assert.True(subgroup.Partial); } [Fact]