diff --git a/CHANGELOG.md b/CHANGELOG.md index dff1a25..91d5093 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes are documented here. The format follows [Keep a Changelog](h and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html); versions before 1.0 may change the API between minor releases. +## [Unreleased] + +### Added + +- `CompressedPool.ReadSubgroups` and `PoolSubgroup`: the object records the object database keeps inside a pool — + BL-tagged subgroups with the object's class id (`Kind`), `Version`, `Flags`, `ObjectId` (a GUID when the header + carries one), the payload's storage (`Mode`: raw or zero mask) and the decoded payload (`Data`). The headers follow + the file's block layout (a u16 packed size for size classes 0..6, a u32 for 7..14); a zero-mask stream that ends + before its declared unpacked size leaves the tail of the buffer zero. Mode 2 (LZ) subgroups have not been met inside + the pools examined so far and are reported with an empty payload. The records are found by scanning for the BL tag + and accepting the ones whose header parses; a rejected candidate is retried one byte on. Observed payloads store text + as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. + +- `PlaDocument.FindCompressedPools` and `CompressedPool`: the compressed storage pools of the object database that the + file holds in the gaps between blocks, each starting with a 16-byte compression sign, a declared decompressed total + and its blocks (each memcpy or LZ77-triple coded, at most one megabyte and change of packed payload per block). The + object database keeps its groups, journals and library data in these pools; a pool is not a block and does not take + part in the block index. `FindCompressedPools` finds every pool whose header and block headers parse and whose + declared sizes tile the total (computed on each call; the scan reads the file in windows and skips the indexed block + payloads, so a sign inside them is never returned as a pool); `Decode` produces the decoded bytes on request. The + layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample + files. + ## [0.1.0] - 2026-11-12 ### Added diff --git a/docs/public-api.md b/docs/public-api.md index f3e6b94..aadea5b 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -1,8 +1,9 @@ # Public API The `PlaIO` package is the supported surface listed here; every public type is in the namespace `PlaIO`. Everything else -(the block scanner, the header parser, the payload decoder, the directory search, the PNG chunk walker) is `internal` in -`PlaIO.Internal`; it can change in any release and is visible only to the test project through `InternalsVisibleTo`. +(the block scanner, the header parser, the payload decoder, the directory search, the pool decoder, the subgroup parser, +the PNG chunk walker) is `internal` in `PlaIO.Internal`; it can change in any release and is visible only to the test +project through `InternalsVisibleTo`. ## File kind @@ -30,6 +31,33 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `ObjectDirectory` | A run of directory records inside a raw block: `Block`, `RecordOffset` (payload offset of the first record), `Phase` (`RecordOffset % 30`), `Records`, `AbuttingPairs`, `OverlappingPairs`, `DuplicatePositions`. The run is an estimate and can be shorter than the real directory: it stops after more than 8 consecutive records without an abutting neighbour. | | `ObjectDirectoryRecord` | The 30-byte record `[u16 class][u64 pos][u32 count][GUID16]` as `ClassNumber`, `Position`, `Count`, `ObjectId`. The units of position and count are not documented, and the class number is not mapped to a class name. | +## Compressed pools (the object database) + +The object database keeps its groups, journals and library data in compressed pools that sit in the gaps between blocks. +A pool starts with a 16-byte compression sign, then the declared decompressed total and the block count (both unsigned +64-bit little endian), then that many blocks, each a u32 compressed size followed by the block: a flag byte (bit 0: the +payload is LZ77-triple coded, otherwise copied; bit 1: the sizes are 32-bit, otherwise single bytes), the packed size +(equal to the compressed size), the unpacked size and the payload. The blocks decode to exactly the declared total. The +layout and the LZ77-triple codec follow the format notes of this repository and are confirmed against the sample files. + +| Member | Use | +|---|---| +| `PlaDocument.FindCompressedPools()` | Every pool whose header and block headers parse and whose declared sizes tile the total, in file order. Recomputed on each call; the scan reads the file in windows and skips the indexed block payloads, so a sign inside them is never returned as a pool. | +| `CompressedPool.Offset` | File offset of the pool's compression sign. | +| `CompressedPool.End` | File offset past the last block of the pool. | +| `CompressedPool.TotalDecompressed` | The declared size of the decoded pool in bytes. | +| `CompressedPool.BlockCount` | The number of blocks; at most one megabyte and change of packed payload per block. | +| `CompressedPool.Decode()` | The decoded bytes: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache. Throws `InvalidDataException` when a block does not decode or the blocks do not add up to the total, and `ObjectDisposedException` when the document is disposed. | +| `CompressedPool.ReadSubgroups()` | The object records inside the pool: BL-tagged subgroups whose headers parse, with their payloads decoded. Recomputed on each call. A rejected BL candidate is retried one byte on. | +| `PoolSubgroup.Offset` | Offset of the subgroup header inside the decoded pool. | +| `PoolSubgroup.Kind` | The object's class id (the kind field of the BL header). | +| `PoolSubgroup.Version` | The header's version byte, 0 or 1. | +| `PoolSubgroup.Flags` | The header's flags: the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the payload modes, the low 4 bits are the packed size class. | +| `PoolSubgroup.Mode` | The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but not decoded; it has not been met inside the pools examined so far. | +| `PoolSubgroup.ObjectId` | The object's GUID, present when flag 0x20 is set. | +| `PoolSubgroup.Partial` | True when decoding stopped at a truncated literal or the payload could not be decoded (unsupported mode or size). A zero-mask stream ending at a group boundary has valid zero padding. | +| `PoolSubgroup.Data` | The decoded payload; empty for mode 2 subgroups. A zero-mask stream that ends before its declared unpacked size leaves the tail of the buffer zero. Observed payloads store text as a u32 length followed by UTF-16 little-endian data and numbers as little-endian doubles and integers. | + ## BIMx file (`.bimx`) | Member | Use | @@ -46,7 +74,8 @@ The `PlaIO` package is the supported surface listed here; every public type is i `PlaDocument` and `BimxDocument` never load the whole file into memory: the block index is built by scanning through the stream once, at `Open`, and payloads and the cover are read on request. A document is not thread-safe; use one per thread. -After `Dispose`, `ReadPayload`, `FindDirectories` and `ReadCover` throw `ObjectDisposedException`. +After `Dispose`, `ReadPayload`, `FindDirectories`, `FindCompressedPools`, `CompressedPool.Decode`, +`CompressedPool.ReadSubgroups` and `ReadCover` throw `ObjectDisposedException`. ## Diagnostics diff --git a/src/PlaIO/CompressedPool.cs b/src/PlaIO/CompressedPool.cs new file mode 100644 index 0000000..ccc3771 --- /dev/null +++ b/src/PlaIO/CompressedPool.cs @@ -0,0 +1,47 @@ +namespace PlaIO; + +/// A compressed storage pool of the ODB object database that the file holds in a gap between blocks: the +/// compression sign, the declared decompressed total and the blocks (each memcpy or LZ77-triple coded). The object +/// database keeps its groups, journals and library data in such pools; a pool is not a block and does not take part in +/// the block index or the payloads. The decoded bytes are produced on request by ; the cost of a +/// call grows with the pool size. The layout and the codec follow the format notes of this repository and are confirmed +/// against the sample files. +public sealed class CompressedPool +{ + private readonly PlaDocument _document; + + internal CompressedPool(PlaDocument document, long offset, long end, long total, int blockCount) + { + _document = document; + Offset = offset; + End = end; + TotalDecompressed = total; + BlockCount = blockCount; + } + + /// File offset of the pool's compression sign. + public long Offset { get; } + + /// File offset past the last block of the pool. + public long End { get; } + + /// The declared size of the decoded pool in bytes; the blocks decode to exactly this size. + public long TotalDecompressed { get; } + + /// The number of blocks; at most one megabyte and change of packed payload per block. + public int BlockCount { get; } + + /// Decodes the pool: every block memcpy or LZ77-triple decoded, concatenated. The call does not cache; each + /// call decodes again. + /// A block does not decode or the blocks do not add up to + /// . + /// The document is disposed. + public byte[] Decode() => _document.DecodePool(Offset); + + /// Decodes the pool and reads its object records: the BL-tagged subgroups whose headers parse, with their + /// payloads decoded (raw or zero mask; mode 2 is reported with an empty payload). Recomputed on each call; a + /// rejected BL candidate is retried one byte on, so a short false header can skip over a real record. + /// The pool does not decode. + /// The document is disposed. + public IReadOnlyList ReadSubgroups() => Internal.SubgroupParser.Parse(Decode()); +} diff --git a/src/PlaIO/Internal/PayloadDecoder.cs b/src/PlaIO/Internal/PayloadDecoder.cs index 4d9e88d..e9bc841 100644 --- a/src/PlaIO/Internal/PayloadDecoder.cs +++ b/src/PlaIO/Internal/PayloadDecoder.cs @@ -2,16 +2,53 @@ namespace PlaIO.Internal; internal static class PayloadDecoder { + /// How a zero-mask stream ended relative to the destination buffer. + public enum ZeroMaskEnd + { + /// The destination buffer is full. + Complete, + /// The stream ended exactly at a group boundary before the buffer is full; the tail of the buffer stays + /// zero. Valid for the pool subgroups, whose declared size is the buffer size. + StreamEnd, + /// The stream ended inside a group: a mask byte was read whose set bit takes a stream byte that is not + /// there. A truncated literal, never a valid padding. + MidGroup, + } + /// Mode 1: a stream of groups, each a mask byte followed by one byte per set bit. Bits are read from the least /// significant; a set bit takes the next stream byte, a clear bit writes 0x00. Decoding ends when - /// is full, so bits of the last mask past the end consume nothing. Returns the number of source bytes consumed; bytes - /// left after that are allowed. + /// is full, so bits of the last mask past the end consume nothing; the status tells how the + /// stream ended when is not full. Returns the number of source bytes consumed; bytes left after + /// that are allowed. + public static int DecodeZeroMask(ReadOnlySpan src, Span dst, out ZeroMaskEnd end) + { + return DecodeCore(src, dst, out end, out _); + } + + /// Mode 1 for the file blocks: the stream must fill the buffer exactly; a stream that ends before that — at a + /// group boundary or inside a group — does not decode. Returns the number of source bytes consumed; bytes left after + /// that are allowed. + /// The stream ends before the buffer is full. public static int DecodeZeroMask(ReadOnlySpan src, Span dst) { - int consumed = 0, written = 0; + int consumed = DecodeCore(src, dst, out _, out var written); + if (written < dst.Length) + throw Short(consumed, src.Length, written, dst.Length); + return consumed; + } + + private static int DecodeCore(ReadOnlySpan src, Span dst, out ZeroMaskEnd end, out int written) + { + int consumed = 0; + written = 0; + end = ZeroMaskEnd.Complete; while (written < dst.Length) { - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) + { + end = ZeroMaskEnd.StreamEnd; + return consumed; + } int mask = src[consumed++]; for (int bit = 0; bit < 8 && written < dst.Length; bit++) { @@ -20,7 +57,11 @@ public static int DecodeZeroMask(ReadOnlySpan src, Span dst) dst[written++] = 0; continue; } - if (consumed >= src.Length) throw Short(consumed, src.Length, written, dst.Length); + if (consumed >= src.Length) + { + end = ZeroMaskEnd.MidGroup; + return consumed; + } dst[written++] = src[consumed++]; } } diff --git a/src/PlaIO/Internal/PoolDecoder.cs b/src/PlaIO/Internal/PoolDecoder.cs new file mode 100644 index 0000000..35de0db --- /dev/null +++ b/src/PlaIO/Internal/PoolDecoder.cs @@ -0,0 +1,294 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// The compressed storage pools that live in the gaps between blocks. A pool starts with the 16-byte compression +/// sign, then the decompressed total and the block count (both unsigned 64-bit little endian), then that many blocks, each +/// a u32 compressed size followed by the block. A block is a flag byte (bit 0: the payload is LZ77-triple coded, otherwise +/// copied; bit 1: the sizes are 32-bit, otherwise bytes), the packed size (equal to the block's compressed size), the +/// unpacked size and the payload. The blocks decode to exactly the declared total. The layout and the codec follow the +/// format notes of this repository and are confirmed against the sample files. +internal static class PoolDecoder +{ + /// The compression sign that opens every compressed pool. + public static ReadOnlySpan Sign => [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + /// At most one megabyte and change of packed payload per block; the reader allocates a buffer of this size + /// plus a byte for the block. + public const int MaxCompressedBlock = 1048976; + + /// The pools of the stream: every compression sign whose header parses and whose block headers tile the + /// declared total inside the file. The scan continues past an accepted pool; a rejected sign is retried one byte on. + /// The ranges are skipped entirely: a sign inside them is not considered (the document + /// passes the byte ranges of its indexed blocks, whose payloads can carry the sign as data). Returns the pool + /// descriptions in file order. + public static List<(long Offset, long End, ulong Total, int BlockCount)> Find(Stream stream, long length, IReadOnlyList<(long Start, long End)>? excluded) + { + var pools = new List<(long Offset, long End, ulong Total, int BlockCount)>(); + var window = new byte[1 << 20]; + long pos = 0; + while (pos + Sign.Length <= length) + { + // Step over an excluded range whole: its signs are data, not pool starts. + if (excluded is not null) + foreach (var (start, end) in excluded) + { + if (pos >= start && pos < end) pos = end; + } + if (pos + Sign.Length > length) break; + int count = (int)Math.Min(window.Length, length - pos); + StreamReads.ReadExactlyAt(stream, pos, window.AsSpan(0, count), "the pool scan"); + int at = IndexOfSign(window.AsSpan(0, count)); + if (at < 0 || pos + at + Sign.Length > length) + { + // The next scan start keeps a full sign inside the next window; a sign can straddle the boundary. + pos = count == window.Length ? pos + count - (Sign.Length - 1) : length; + continue; + } + long signAt = pos + at; + if (Inside(excluded, signAt)) + { + pos = signAt + 1; + continue; + } + if (TryValidate(stream, length, signAt, out var pool)) + { + pools.Add(pool); + pos = pool.End; + } + else + { + pos = signAt + 1; + } + } + return pools; + } + + /// Decodes the pool at : its blocks, each memcpy or LZ77-triple, concatenated. + /// Throws when the blocks do not decode or do not add up to the declared total. + public static byte[] Decode(Stream stream, long offset, long length) + { + var (total, blocks) = ReadBlocks(stream, offset, length); + if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit"); + var output = new byte[total]; + long written = 0; + foreach (var (payloadAt, packed, unpacked) in blocks) + { + var payload = new byte[packed]; + StreamReads.ReadExactlyAt(stream, payloadAt, payload, $"pool at 0x{offset:x}"); + int flag = payload[0]; + int header = (flag & 0b10) != 0 ? 9 : 3; + int produced = (flag & 1) != 0 + ? DecodeTriple(payload, output.AsSpan((int)written, (int)unpacked), (int)unpacked) + : CopyPayload(payload[header..], output.AsSpan((int)written, (int)unpacked)); + if (produced != (int)unpacked) + throw new InvalidDataException($"Pool at 0x{offset:x}: block at 0x{payloadAt:x} produced {produced} of {unpacked} bytes."); + written += unpacked; + } + if (written != (long)total) + throw new InvalidDataException($"Pool at 0x{offset:x}: blocks produced {written} of {total} bytes."); + return output; + } + + /// The block layout of the pool whose sign sits at : the u64 total and block count + /// after the sign, then one entry per block with its payload offset, packed and unpacked size. The declared total can + /// exceed the container (the codec is compressing); only the array limit and the file bounds of every block header + /// and payload are checked. The sizes must tile the declared total exactly. + private static (ulong Total, List<(long PayloadAt, int Packed, int Unpacked)> Blocks) ReadBlocks(Stream stream, long offset, long length) + { + var header = new byte[16]; + if (offset + Sign.Length + header.Length > length) throw Bad(offset, "its header runs past the file"); + StreamReads.ReadExactlyAt(stream, offset + Sign.Length, header, $"pool at 0x{offset:x}"); + ulong total = BinaryPrimitives.ReadUInt64LittleEndian(header); + ulong blockCount = BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(8)); + if (total > int.MaxValue) throw Bad(offset, $"its declared total {total} exceeds the array limit"); + var blocks = new List<(long, int, int)>(); + long pos = offset + Sign.Length + header.Length; + ulong sum = 0; + for (ulong i = 0; i < blockCount; i++) + { + var sizeBytes = new byte[4]; + if (pos + 4 > length) throw Bad(offset, "the block list runs past the file"); + StreamReads.ReadExactlyAt(stream, pos, sizeBytes, $"pool at 0x{offset:x}"); + uint packed = BinaryPrimitives.ReadUInt32LittleEndian(sizeBytes); + pos += 4; + if (packed < 3 || packed > MaxCompressedBlock || pos + packed > length) throw Bad(offset, $"block {i} declares {packed} packed bytes"); + var flagBytes = new byte[1]; + StreamReads.ReadExactlyAt(stream, pos, flagBytes, $"pool at 0x{offset:x}"); + int flag = flagBytes[0]; + bool big = (flag & 0b10) != 0; + int headerLength = big ? 9 : 3; + if (packed < headerLength) throw Bad(offset, $"block {i} declares {packed} packed bytes, less than its {headerLength}-byte header"); + var head = new byte[headerLength]; + StreamReads.ReadExactlyAt(stream, pos, head, $"pool at 0x{offset:x}"); + uint declaredPacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(1)) : head[1]; + if (declaredPacked != packed) throw Bad(offset, $"block {i} header declares {declaredPacked} packed bytes, the pool lists {packed}"); + uint unpacked = big ? BinaryPrimitives.ReadUInt32LittleEndian(head.AsSpan(5)) : head[2]; + if (sum + unpacked > total) throw Bad(offset, $"block {i} overruns the declared total"); + blocks.Add((pos, (int)packed, (int)unpacked)); + sum += unpacked; + pos += packed; + } + if (sum != total) throw Bad(offset, $"its blocks add up to {sum}, not the declared {total}"); + return (total, blocks); + } + + /// Whether the header and every block header of the pool at parse and the declared + /// sizes tile the total inside the file. + private static bool TryValidate(Stream stream, long length, long offset, out (long Offset, long End, ulong Total, int BlockCount) pool) + { + pool = default; + try + { + var (total, blocks) = ReadBlocks(stream, offset, length); + pool = (offset, blocks.Count > 0 ? blocks[^1].PayloadAt + blocks[^1].Packed : offset + Sign.Length + 16, total, blocks.Count); + return true; + } + catch (InvalidDataException) + { + return false; + } + } + + /// The first offset of the compression sign in , or -1. + private static int IndexOfSign(ReadOnlySpan window) + { + int at = 0; + while (true) + { + int next = window[at..].IndexOf(Sign[0]); + if (next < 0) return -1; + at += next; + if (at + Sign.Length > window.Length) return -1; + if (window.Slice(at, Sign.Length).SequenceEqual(Sign)) return at; + at++; + } + } + + /// Whether falls inside one of the excluded ranges. + private static bool Inside(IReadOnlyList<(long Start, long End)>? excluded, long at) + { + if (excluded is null) return false; + foreach (var (start, end) in excluded) + if (start <= at && at < end) return true; + return false; + } + + /// The payload of a memcpy block (the bytes after the block header, already sliced), at most + /// bytes. + private static int CopyPayload(ReadOnlySpan block, Span dst) + { + int source = Math.Min(block.Length, dst.Length); + block[..source].CopyTo(dst); + return source; + } + + /// The LZ77-triple codec. The stream is a little-endian flag word (a 1 starts a new word) and then, per flag + /// bit: a set bit takes a token whose low 16 bits name a slot of the pointer table (the top 12 bits, shifted right by + /// 4) and a length ((the low 4 bits) + 2, or a following byte when 0); the copy reads from that slot and runs in + /// 3-byte steps over unsigned 32-bit reads, so the last step may write up to two bytes past the length. A clear bit + /// takes a token whose first four bytes go to the output, of which the literal-length table keeps the first. The last + /// ten bytes go out one by one. Every position up to a few bytes behind the output position is registered into the + /// 4096-slot table, hashed from the four output bytes at position+1. An out-of-range slot or length stops the + /// stream. + private static int DecodeTriple(ReadOnlySpan block, Span output, int unpacked) + { + var table = new long[4096]; + Array.Fill(table, -1); + + bool big = (block[0] & 0b10) != 0; + int packedSize = big ? (int)BinaryPrimitives.ReadUInt32LittleEndian(block[1..]) : block[1]; + int endIn = packedSize; // the last valid input index is endIn - 1 + int src = 3 + (big ? 6 : 0); // the payload starts after the header + int outPos = 0; + uint flags = 1; + long cursor = -1; + int outEnd = unpacked - 1; + + uint token = 0; + while (true) + { + while (true) + { + if (flags == 1) + { + if (src + 3 > endIn - 1) return outPos; + flags = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src)); + src += 4; + } + if (src + 3 > endIn - 1) return outPos; + token = BinaryPrimitives.ReadUInt32LittleEndian(block.Slice(src)); + if ((flags & 1) == 0) break; + + flags >>= 1; + src += 2; + long source = table[(ushort)token >> 4]; + int length; + int low = (int)(token & 0xF); + if (low != 0) length = low + 2; + else + { + length = block[src]; + src += 1; + } + if (source < 0 || source > outPos - 3 || length > outEnd - outPos - 3) return outPos; + int step = 0; + while (step < length) + { + // One 32-bit read per 3-byte step; the tail may run up to two bytes past the length. + output[outPos + step] = output[(int)source + step]; + output[outPos + step + 1] = output[(int)source + step + 1]; + output[outPos + step + 2] = output[(int)source + step + 2]; + output[outPos + step + 3] = output[(int)source + step + 3]; + step += 3; + } + outPos += length; + Register(table, ref cursor, outPos - length, output); + cursor = outPos - 1; + } + + if (outPos >= outEnd - 10) break; + output[outPos] = (byte)token; + output[outPos + 1] = (byte)(token >> 8); + output[outPos + 2] = (byte)(token >> 16); + output[outPos + 3] = (byte)(token >> 24); + int literals = LiteralLengths[flags & 0xF]; + outPos += literals; + flags >>= literals; + src += literals; + Register(table, ref cursor, outPos - 3, output); + } + + while (outPos <= outEnd) + { + uint f = flags; + if (flags == 1) flags = 0x80000000; + int at = f == 1 ? src + 4 : src; + if (at >= endIn) return outPos; + output[outPos] = block[at]; + src = at + 1; + outPos++; + flags >>= 1; + } + Register(table, ref cursor, outEnd - 3, output); + return unpacked; + } + + private static readonly int[] LiteralLengths = [4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0]; + + /// Registers every output position below that the cursor has not reached yet: the + /// hash of the four output bytes at position+1 indexes the slot that stores the position. + private static void Register(long[] table, ref long cursor, long limit, ReadOnlySpan output) + { + while (cursor < limit) + { + int at = (int)(cursor + 1); + uint word = (uint)(output[at] | output[at + 1] << 8 | output[at + 2] << 16 | output[at + 3] << 24); + int hash = (int)(((ushort)word ^ (ushort)(word >> 12)) & 0xFFF); + table[hash] = cursor + 1; + cursor++; + } + } + + private static InvalidDataException Bad(long offset, string what) => new($"Pool at 0x{offset:x}: {what}."); +} diff --git a/src/PlaIO/Internal/SubgroupParser.cs b/src/PlaIO/Internal/SubgroupParser.cs new file mode 100644 index 0000000..f9e4449 --- /dev/null +++ b/src/PlaIO/Internal/SubgroupParser.cs @@ -0,0 +1,96 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// The objects the object database keeps inside a compressed pool: BL-tagged records that follow the same header +/// layout as the file's blocks — tag, id, kind (the object's class id), version, flags (marker 0x8080, optional GUID, +/// mode bits), the packed size (a u16 for size classes 0..6, a u32 for 7..14, a u64 for 15) and, for modes 1 and 2, the +/// unpacked size as a u32. A mode 0 payload is stored raw; a mode 1 payload is a zero mask like the file blocks'; a mode +/// 2 payload is LZ-token coded — not met in the pools examined so far, reported with an empty payload. The records are +/// found by scanning for the tag and accepting the ones whose header parses; a candidate whose header cannot be read is +/// retried one byte on, so the scan never stops before the end of the pool. A mode 1 stream that ends exactly at a group +/// boundary before the declared unpacked size is full leaves the tail of the buffer zero; a stream that ends inside a +/// group returns the decoded prefix with zero padding and marks the record as partial. +internal static class SubgroupParser +{ + public static List Parse(byte[] pool) + { + var result = new List(); + int pos = 0; + while (pos + 11 <= pool.Length) + { + if (pool[pos] != 0x42 || pool[pos + 1] != 0x4C) { pos++; continue; } // "BL" + ushort flags = BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 9)); + if ((flags & FdbFormat.MarkerMask) != FdbFormat.Marker) { pos++; continue; } + int q = pos + 11; + if ((flags & FdbFormat.FlagGuid) != 0) + { + if (q + 16 > pool.Length) { pos++; continue; } + q += 16; + } + int sizeClass = flags & 0xF; + int width = sizeClass <= 6 ? 2 : sizeClass <= 14 ? 4 : 8; + if (q + width > pool.Length) { pos++; continue; } + long packed = width switch + { + 2 => BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(q)), + 4 => BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)), + _ => unchecked((long)BinaryPrimitives.ReadUInt64LittleEndian(pool.AsSpan(q))), + }; + q += width; + var mode = (flags & FdbFormat.FlagZeroMask) != 0 ? PlaBlockMode.ZeroMask + : (flags & FdbFormat.FlagLz) != 0 ? PlaBlockMode.Lz + : PlaBlockMode.Raw; + long unpacked = packed; + if (mode != PlaBlockMode.Raw) + { + if (q + 4 > pool.Length) { pos++; continue; } + unpacked = BinaryPrimitives.ReadUInt32LittleEndian(pool.AsSpan(q)); + q += 4; + } + if (packed < 0 || packed > pool.Length - q || unpacked < 0 || unpacked > (long)pool.Length * 32) { pos++; continue; } + + Guid? objectId = null; + if ((flags & FdbFormat.FlagGuid) != 0) + objectId = new Guid(pool.AsSpan(pos + 11, 16)); + + var data = DecodePayload(pool.AsSpan(q, (int)packed), mode, unpacked, out var partial); + result.Add(new PoolSubgroup(pos, BinaryPrimitives.ReadUInt16LittleEndian(pool.AsSpan(pos + 6)), + pool[pos + 8], flags, mode, objectId, data, partial)); + pos = q + (int)packed; + } + return result; + } + + private static byte[] DecodePayload(ReadOnlySpan body, PlaBlockMode mode, long unpacked, out bool partial) + { + if (unpacked > int.MaxValue) + { + partial = true; + return []; + } + switch (mode) + { + case PlaBlockMode.Raw: + { + partial = false; + var data = new byte[unpacked]; + body[..Math.Min(body.Length, data.Length)].CopyTo(data); + return data; + } + case PlaBlockMode.ZeroMask: + { + // The declared size is the buffer size: a stream that ends exactly at a group boundary leaves the tail + // of the buffer zero, but a stream that ends inside a group is a truncated literal — the record is + // damaged, its tail is reconstructed with zeros and marked Partial. + var data = new byte[unpacked]; + PayloadDecoder.DecodeZeroMask(body, data, out var end); + partial = end == PayloadDecoder.ZeroMaskEnd.MidGroup; + return data; + } + default: + partial = true; + return []; // Mode 2 (LZ) is not met in the pools examined so far. + } + } +} diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 4cc7ad1..3348319 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -142,6 +142,29 @@ public IReadOnlyList FindDirectories() return found.AsReadOnly(); } + /// Finds the compressed storage pools of the object database: every 16-byte compression sign in the file + /// whose header and block headers parse and whose declared sizes tile the total. These pools sit in the gaps between + /// blocks; the byte ranges of the indexed blocks are excluded, so a sign that a block payload happens to carry is + /// never returned as a pool. Computed on each call; the scan reads the file in windows and then the block headers of + /// each candidate. + public IReadOnlyList FindCompressedPools() + { + ObjectDisposedException.ThrowIf(_disposed, this); + // The indexed block payloads are excluded whole: their bytes are data, and a compression sign inside them is + // content, not a pool start. + var excluded = Blocks.Select(b => (b.PayloadOffset, b.End)).ToList(); + return PoolDecoder.Find(_stream, Length, excluded) + .Select(p => new CompressedPool(this, p.Offset, p.End, (long)p.Total, p.BlockCount)) + .ToList() + .AsReadOnly(); + } + + internal byte[] DecodePool(long offset) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return PoolDecoder.Decode(_stream, offset, Length); + } + /// Releases the stream unless the document was opened with leaveOpen. public void Dispose() { diff --git a/src/PlaIO/PoolSubgroup.cs b/src/PlaIO/PoolSubgroup.cs new file mode 100644 index 0000000..a21504d --- /dev/null +++ b/src/PlaIO/PoolSubgroup.cs @@ -0,0 +1,49 @@ +namespace PlaIO; + +/// One object record inside a compressed pool: a BL-tagged header with the object's class id, a GUID and a +/// payload decoded from the pool's compression (raw or zero mask). The meaning of the class id and of the payload's +/// fields is not documented; observed payloads store text as a u32 length followed by UTF-16 little-endian data and +/// numbers as little-endian doubles and integers. +public sealed class PoolSubgroup +{ + internal PoolSubgroup(long offset, int kind, int version, int flags, PlaBlockMode mode, Guid? objectId, byte[] data, bool partial) + { + Offset = offset; + Kind = kind; + Version = version; + Flags = flags; + Mode = mode; + ObjectId = objectId; + Data = data; + Partial = partial; + } + + /// Offset of the subgroup header inside the decoded pool (see ). + public long Offset { get; } + + /// The object's class id (the kind field of the BL header). Values seen in real files run from small ids to + /// 0xFFDC, which the table of contents uses as its free-block marker. + public int Kind { get; } + + /// The header's version byte, 0 or 1. + public int Version { get; } + + /// The header's flags: bit 15..14 pair is the 0x8080 marker, 0x20 marks the GUID, 0x100 and 0x200 select the + /// payload modes, the low 4 bits are the packed size class. + public int Flags { get; } + + /// The payload's storage: raw (0) or zero mask (1). Mode 2 (LZ) is reported but its payload is not decoded; + /// it has not been met inside the pools examined so far. + public PlaBlockMode Mode { get; } + + /// The object's GUID, present when flag 0x20 is set. + public Guid? ObjectId { get; } + + /// The decoded payload. For mode 2 subgroups this is empty. + public byte[] Data { get; } + + /// Whether decoding was incomplete or unavailable. A truncated zero-mask literal leaves a + /// zero-filled tail in . Unsupported mode 2 and payloads too large to decode have empty data. + /// A zero-mask stream that ends at a group boundary may have a valid zero-filled tail with this property false. + public bool Partial { get; } +} diff --git a/tests/PlaIO.Tests/CompressedPoolTests.cs b/tests/PlaIO.Tests/CompressedPoolTests.cs new file mode 100644 index 0000000..0405d5d --- /dev/null +++ b/tests/PlaIO.Tests/CompressedPoolTests.cs @@ -0,0 +1,193 @@ +using System.Buffers.Binary; +using Xunit; + +namespace PlaIO.Tests; + +public class CompressedPoolTests +{ + private static byte[] PoolOf(params byte[][] payloads) => Synthetic.Fdb(Synthetic.Pool(payloads)); + + [Fact] + public void AFileWithoutSignsFindsNoPools() + { + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.RandomBytes(new Random(1), 4096)))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void OneBigMemCpyBlockDecodesToItsPayload() + { + var payload = Enumerable.Range(0, 500).Select(i => (byte)(i * 7)).ToArray(); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(payload))); + var pools = document.FindCompressedPools(); + Assert.Equal([1], pools.Select(p => p.BlockCount).ToArray()); + var pool = Assert.Single(pools); + Assert.Equal(payload.Length, pool.TotalDecompressed); + Assert.Equal(payload, pool.Decode()); + } + + [Fact] + public void SmallAndBigBlocksDecodeToTheirPayloadsConcatenated() + { + var first = Synthetic.RandomBytes(new Random(2), 100); + var second = Synthetic.RandomBytes(new Random(3), 300); + var bytes = Synthetic.Fdb(Synthetic.Pool([first], small: true), Synthetic.Pool([second])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pools = document.FindCompressedPools(); + Assert.Equal(2, pools.Count); + Assert.Equal(first, pools[0].Decode()); + Assert.Equal(second, pools[1].Decode()); + // The file offsets do not overlap: the second pool starts past the first one's last block. + Assert.True(pools[0].End <= pools[1].Offset); + } + + [Fact] + public void AZeroBlockPoolIsValidAndDecodesToNothing() + { + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), new byte[16]); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(0, pool.TotalDecompressed); + Assert.Empty(pool.Decode()); + } + + [Fact] + public void ATruncatedPoolIsNotFound() + { + var bytes = Synthetic.Pool([Synthetic.RandomBytes(new Random(4), 200)]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(bytes[..^32]))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void ABlockSizeMismatchIsNotFound() + { + // Flip one byte of the block's declared packed size (the pool holds the sign, the two u64 sizes, the u32 + // compressed size and the flag byte): it stops matching the compressed size the pool lists. + var pool = Synthetic.Pool([Synthetic.RandomBytes(new Random(5), 64)]); + var corrupt = pool.ToArray(); + corrupt[16 + 16 + 4 + 1] ^= 0xFF; + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(corrupt))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void LZBlockThatDoesNotDecodeFindsButDecodeThrows() + { + // A big block that declares LZ77 coding (bit 0 set, flag 0x47) with an impossible token stream: Find accepts the + // shape (the block headers tile the total), Decode fails. + var block = new byte[9 + 16]; + block[0] = 0x47; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 16); + 0x00FF00FFu.CopyToBlock(block, 9); + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(16), Le64(1), Le32((uint)block.Length), block); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Throws(pool.Decode); + } + + [Fact] + public void ADeclaredTotalPastTheContainerIsStillAccepted() + { + // The codec is compressing: a small pool can declare a total bigger than the container itself. The block header + // parses inside the file and its declared unpacked size tiles the total, so Find accepts the pool even though + // the declared total is past the end of the file; only the array limit and the per-block bounds are checked. + var block = new byte[9 + 16]; + block[0] = 0x47; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), 128); + 0x00FF00FFu.CopyToBlock(block, 9); + var bytes = Synthetic.Fdb(Synthetic.PoolSign(), Le64(128), Le64(1), Le32((uint)block.Length), block); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(128, pool.TotalDecompressed); + Assert.True(pool.TotalDecompressed > document.Length); + } + + [Fact] + public void ASignInsideAnIndexedBlockPayloadIsNotAPool() + { + // A raw block whose payload carries a compression sign and a parseable pool header: the sign is data, not a + // pool start, and Find must not return it. + var payload = Synthetic.Pool([Synthetic.RandomBytes(new Random(8), 64)]); + var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(block))); + Assert.Empty(document.FindCompressedPools()); + } + + [Fact] + public void APoolInTheGapAfterAnIndexedBlockIsFound() + { + // The same container with the pool moved out of the block: the block payload is a filler, the pool follows it. + var payload = Synthetic.RandomBytes(new Random(9), 64); + var block = Synthetic.Block(PlaBlockMode.Raw, payload, kind: 0x015B); + var bytes = Synthetic.Fdb(block, Synthetic.Pool([Synthetic.RandomBytes(new Random(10), 64)])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.True(pool.Offset >= 32 + payload.Length, $"pool at 0x{pool.Offset:x} overlaps the block"); + } + + [Fact] + public void DecodeAfterDisposeThrows() + { + var document = PlaDocument.Open(new MemoryStream(PoolOf([1, 2, 3]))); + var pool = Assert.Single(document.FindCompressedPools()); + document.Dispose(); + Assert.Throws(pool.Decode); + Assert.Throws(document.FindCompressedPools); + } + + [Fact] + public void ASignAcrossTheWindowBoundaryIsStillFound() + { + // Put the sign where the 1 MiB scan window ends eight bytes after its start: the next window must overlap it. + var filler = Synthetic.RandomBytes(new Random(6), (1 << 20) - 8); + var bytes = Synthetic.Fdb(filler, Synthetic.Pool([Synthetic.RandomBytes(new Random(7), 300)])); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var pool = Assert.Single(document.FindCompressedPools()); + Assert.Equal(300, pool.TotalDecompressed); + Assert.Equal(300, pool.Decode().Length); + } + + [SkippableFact] + public void EverySampleFindsPoolsAndThePoolsOfTheSmallerOnesDecode() + { + var corpus = new SampleCorpus(); + try + { + foreach (var sample in corpus.Fdb) + { + var pools = sample.Document.FindCompressedPools(); + bool decodeAll = sample.Document.Length < 32 << 20; + foreach (var pool in decodeAll ? pools : pools.Take(8)) + Assert.True(pool.Decode().Length == pool.TotalDecompressed, + $"{sample.Name}: pool at 0x{pool.Offset:x} does not decode to {pool.TotalDecompressed}"); + } + } + finally + { + corpus.Dispose(); + } + } + + private static byte[] Le32(uint value) + { + var bytes = new byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return bytes; + } + + private static byte[] Le64(ulong value) + { + var bytes = new byte[8]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + return bytes; + } +} + +file static class CopyExtensions +{ + public static void CopyToBlock(this uint value, byte[] block, int at) => + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(at), value); +} diff --git a/tests/PlaIO.Tests/PoolSubgroupTests.cs b/tests/PlaIO.Tests/PoolSubgroupTests.cs new file mode 100644 index 0000000..b9a3cd1 --- /dev/null +++ b/tests/PlaIO.Tests/PoolSubgroupTests.cs @@ -0,0 +1,184 @@ +using System.Buffers.Binary; +using System.Text; +using Xunit; + +namespace PlaIO.Tests; + +public class PoolSubgroupTests +{ + private static byte[] PoolOf(params byte[][] subgroups) => Synthetic.Fdb(Synthetic.Pool([subgroups.SelectMany(s => s).ToArray()])); + + private static byte[] TextPayload(string text) => + BitConverter.GetBytes((uint)text.Length).Concat(Encoding.Unicode.GetBytes(text)).ToArray(); + + [Fact] + public void RawSubgroupIsParsedWithKindGuidAndData() + { + var guid = Guid.NewGuid(); + var payload = Synthetic.RandomBytes(new Random(1), 100); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, payload, PlaBlockMode.Raw, guid)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(0x009B, subgroup.Kind); + Assert.Equal(1, subgroup.Version); + Assert.Equal(guid, subgroup.ObjectId); + Assert.Equal(PlaBlockMode.Raw, subgroup.Mode); + Assert.Equal(payload, subgroup.Data); + Assert.Equal(0, subgroup.Offset); + } + + [Fact] + public void ZeroMaskSubgroupIsDecodedToItsData() + { + var payload = Synthetic.ZeroMask(Synthetic.RandomBytes(new Random(2), 200)); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0045, payload, PlaBlockMode.ZeroMask, unpacked: 200)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(PlaBlockMode.ZeroMask, subgroup.Mode); + Assert.Equal(200, subgroup.Data.Length); + Assert.Equal(Synthetic.RandomBytes(new Random(2), 200), subgroup.Data); + } + + [Fact] + public void AZeroMaskStreamShorterThanTheBufferLeavesTheTailZero() + { + // The declared unpacked size is the buffer size; the stream fills the head and stops. + var payload = Synthetic.ZeroMask([1, 2, 3, 4]); + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, payload, PlaBlockMode.ZeroMask, unpacked: 64)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(64, subgroup.Data.Length); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, subgroup.Data[..4]); + Assert.True(subgroup.Data[4..].All(b => b == 0)); + Assert.False(subgroup.Partial); + } + + [Fact] + public void ATruncatedLiteralIsMarkedPartialAndKeepsTheProducedBytes() + { + // The last mask demands more literals than the stream holds: the record is damaged. The subgroup keeps the + // bytes the stream produced, the tail is reconstructed with zeros, and Partial marks it. + var plain = new byte[] { 1, 2, 3, 4, 5, 6 }; + var full = Synthetic.ZeroMask(plain); // mask 0x3F, six data bytes, ends at a group boundary + // Cut the last two literals off: the mask still demands six, the stream only holds four. + var truncated = full[..(full.Length - 2)]; + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0016, truncated, PlaBlockMode.ZeroMask, unpacked: 6)))); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.True(subgroup.Partial); + Assert.Equal(6, subgroup.Data.Length); + Assert.Equal(new byte[] { 1, 2, 3, 4, 0, 0 }, subgroup.Data); + } + + [Fact] + public void TextPayloadIsReadAsALengthPrefixedUtf16String() + { + using var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x009B, TextPayload("Sample text"), PlaBlockMode.Raw)))); + var data = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()).Data; + int len = BitConverter.ToInt32(data, 0); + Assert.Equal("Sample text", Encoding.Unicode.GetString(data, 4, len * 2)); + } + + [Fact] + public void SubgroupsAreParsedInOrderAndOffsetsArePoolRelative() + { + var first = Synthetic.RandomBytes(new Random(3), 60); + var second = Synthetic.RandomBytes(new Random(4), 30); + var bytes = PoolOf( + Synthetic.Subgroup(0x0003, first, PlaBlockMode.Raw), + Synthetic.Subgroup(0x005D, second, PlaBlockMode.ZeroMask, unpacked: 30)); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroups = document.FindCompressedPools()[0].ReadSubgroups(); + Assert.Equal(2, subgroups.Count); + Assert.Equal(0x0003, subgroups[0].Kind); + Assert.Equal(first, subgroups[0].Data); + Assert.Equal(0x005D, subgroups[1].Kind); + Assert.True(subgroups[1].Offset > subgroups[0].Offset); + } + + [Fact] + public void AMode2SubgroupIsReportedWithoutData() + { + var bytes = PoolOf(Synthetic.Subgroup(0x0045, Synthetic.RandomBytes(new Random(5), 40), PlaBlockMode.Lz, unpacked: 80)); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(PlaBlockMode.Lz, subgroup.Mode); + Assert.Empty(subgroup.Data); + Assert.True(subgroup.Partial); + } + + [Fact] + public void JunkBetweenSubgroupsIsSkipped() + { + var filler = Synthetic.RandomBytes(new Random(6), 50); + // Junk that starts with "BL" but fails the header checks, right before a real subgroup, in one block. + var bytes = PoolOf(new byte[] { 0x42, 0x4C, 0x99 }.Concat(Synthetic.Subgroup(0x0006, filler, PlaBlockMode.Raw)).ToArray()); + using var document = PlaDocument.Open(new MemoryStream(bytes)); + var subgroup = Assert.Single(document.FindCompressedPools()[0].ReadSubgroups()); + Assert.Equal(0x0006, subgroup.Kind); + Assert.Equal(filler, subgroup.Data); + } + + [Fact] + public void ReadSubgroupsAfterDisposeThrows() + { + var document = PlaDocument.Open(new MemoryStream(PoolOf(Synthetic.Subgroup(0x0006, [1, 2, 3], PlaBlockMode.Raw)))); + var pool = Assert.Single(document.FindCompressedPools()); + document.Dispose(); + Assert.Throws(pool.ReadSubgroups); + } + + [SkippableFact] + public void EverySamplePoolParsesItsSubgroups() + { + var corpus = new SampleCorpus(); + try + { + foreach (var sample in corpus.Fdb) + foreach (var pool in sample.Document.FindCompressedPools()) + { + var subgroups = pool.ReadSubgroups(); + Assert.True(subgroups.Count > 0 || pool.TotalDecompressed == 0, + $"{sample.Name}: pool at 0x{pool.Offset:x} of {pool.TotalDecompressed} bytes has no subgroups"); + } + } + finally + { + corpus.Dispose(); + } + } + + [SkippableFact] + public void SomeSampleSubgroupHoldsALengthPrefixedUtf16String() + { + // The payloads store strings as [u32 len][utf16le data]; no sample name or content is asserted, only that the + // corpus contains at least one such string inside a decoded subgroup payload. + var corpus = new SampleCorpus(); + try + { + foreach (var sample in corpus.Fdb) + foreach (var pool in sample.Document.FindCompressedPools()) + foreach (var subgroup in pool.ReadSubgroups()) + { + var data = subgroup.Data; + for (int i = 0; i + 8 <= data.Length; i++) + { + int len = BitConverter.ToInt32(data, i); + if (len is < 1 or > 500 || i + 4 + len * 2 > data.Length) continue; + bool printable = true; + for (int k = 0; k < len * 2 && printable; k += 2) + { + char ch = (char)(data[i + 4 + k] | data[i + 5 + k] << 8); + if (ch != 0 && (ch < 0x20 || ch > 0x7E) && (ch < 0x400 || ch > 0x4FF)) printable = false; + } + if (printable && len >= 2) + { + Assert.True(true, $"found a length-prefixed utf16 string in {sample.Name}"); + return; + } + } + } + Assert.Fail("no length-prefixed utf16 string found in any pool subgroup of any sample"); + } + finally + { + corpus.Dispose(); + } + } +} diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index 75979fd..6bb05b4 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -125,6 +125,74 @@ public static byte[] RandomBytes(Random random, int length) /// A container: the 32-byte header, then the parts from offset 32. public static byte[] Fdb(params byte[][] parts) => Header().Concat(parts.SelectMany(p => p)).ToArray(); + /// A compressed storage pool: the compression sign, the u64 total and block count, then one u32 compressed + /// size plus block per payload. Big memcpy blocks (flag 0x46) unless ; a small block (flag + /// 0x44) keeps its sizes in single bytes and needs at most 252 payload bytes. The big-block flag's other bits match + /// the real files (0x44/0x46/0x47 with bit 2 and bit 6 set). + public static byte[] Pool(byte[][] payloads, bool small = false) + { + var body = new List(); + ulong total = 0; + foreach (var payload in payloads) + { + int headerLength = small ? 3 : 9; + var block = new byte[headerLength + payload.Length]; + block[0] = (byte)(small ? 0x44 : 0x46); // bit 0 clear: memcpy; bit 2 and 6 set like the real files + if (small) + { + block[1] = (byte)block.Length; + block[2] = (byte)payload.Length; + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(1), (uint)block.Length); + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(5), (uint)payload.Length); + } + payload.CopyTo(block, headerLength); + body.AddRange(Le32((uint)block.Length)); + body.AddRange(block); + total += (ulong)payload.Length; + } + var head = new byte[32]; + PoolSign().CopyTo(head); + BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(16), total); + BinaryPrimitives.WriteUInt64LittleEndian(head.AsSpan(24), (ulong)payloads.Length); + return head.Concat(body).ToArray(); + } + + /// The compression sign the pools start with. + public static byte[] PoolSign() => + [0x79, 0xA8, 0xA9, 0x96, 0x45, 0xB4, 0x4A, 0xF3, 0x98, 0x38, 0x33, 0xC3, 0x3A, 0x06, 0x1B, 0x06]; + + /// A BL-tagged subgroup as the pools hold them: tag, id, kind, version, flags (marker 0x8080; GUID flag set + /// when is given), the GUID, the packed size (u16 for size classes 0..6, u32 for 7..14) and, + /// for modes 1 and 2, the u32 unpacked size, then the payload. Mode 1 payloads are zero-mask coded; the declared + /// unpacked size may exceed what the stream produces, the tail stays zero. Mode 2 payloads are reported but not + /// coded here (the tests only check that the parser reports them with an empty payload). + public static byte[] Subgroup(ushort kind, byte[] payload, PlaBlockMode mode = PlaBlockMode.Raw, Guid? guid = null, + uint id = 7, byte version = 1, int unpacked = -1) + { + int sizeClass = SizeClass.Of((ulong)payload.Length); + int modeFlag = mode switch { PlaBlockMode.ZeroMask => 0x100, PlaBlockMode.Lz => 0x200, _ => 0 }; + ushort flags = (ushort)(0x8080 | sizeClass | (guid != null ? 0x20 : 0) | modeFlag); + var bytes = new List(); + bytes.AddRange(System.Text.Encoding.ASCII.GetBytes("BL")); + bytes.AddRange(Le32(id)); + bytes.AddRange(Le16(kind)); + bytes.Add(version); + bytes.AddRange(Le16(flags)); + if (guid != null) bytes.AddRange(guid.Value.ToByteArray()); + if (sizeClass <= 6) bytes.AddRange(Le16((ushort)payload.Length)); + else bytes.AddRange(Le32((uint)payload.Length)); + if (mode != PlaBlockMode.Raw) + { + uint declared = unpacked >= 0 ? (uint)unpacked : (uint)payload.Length; + bytes.AddRange(Le32(declared)); + } + bytes.AddRange(payload); + return bytes.ToArray(); + } + /// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked). public static byte[] Png(params (string Type, byte[] Data)[] chunks) {