diff --git a/interscript-py/scripts/time_corpus_py.py b/interscript-py/scripts/time_corpus_py.py new file mode 100644 index 00000000..49780429 --- /dev/null +++ b/interscript-py/scripts/time_corpus_py.py @@ -0,0 +1,118 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Corpus parse timing for the generated ISC parser (perf work harness). + +Parses every corpus file (maps/*.isc + libs/*.isc) with the generated +parser only, dependencies resolved from the same corpus, and reports the +total wall time. One run = one full corpus pass; invoke N times for a +median. --profile cProfiles a single file instead. + +Usage: + python scripts/time_corpus_py.py # one corpus pass + python scripts/time_corpus_py.py --repeat 3 # three passes, median + python scripts/time_corpus_py.py --profile var-kor +""" + +from __future__ import annotations + +import argparse +import cProfile +import pstats +import statistics +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +PY_SRC = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(PY_SRC)) + +from interscript.isc import parse_isc + + +def corpus_files(maps_root: Path): + files = [] + dep_index = {} + for sub in ("maps", "libs"): + d = maps_root / sub + for f in sorted(d.iterdir()): + if f.suffix == ".isc": + files.append({"rel": f"{sub}/{f.name}", "path": f}) + dep_index[f.stem] = f + return files, dep_index + + +def one_pass(files, dep_index): + cache: dict = {} + + def resolver(target): + if target in cache: + return cache[target] + entry = dep_index.get(target) + if entry is None: + return None + with open(entry, encoding="utf-8") as fh: + doc = parse_isc(fh.read(), entry, resolve_dependency=resolver) + cache[target] = doc + return doc + + t0 = time.perf_counter() + for f in files: + with open(f["path"], encoding="utf-8") as fh: + parse_isc(fh.read(), f["path"], resolve_dependency=resolver) + return time.perf_counter() - t0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--maps-root", default=str(REPO_ROOT.parent / "maps")) + ap.add_argument("--repeat", type=int, default=1) + ap.add_argument("--profile") + args = ap.parse_args() + + maps_root = Path(args.maps_root).resolve() + files, dep_index = corpus_files(maps_root) + if not files: + print(f"no corpus files under {maps_root}", file=sys.stderr) + return 2 + + if args.profile: + target = next(f for f in files if args.profile in f["rel"]) + cache: dict = {} + + def resolver(target): + if target in cache: + return cache[target] + entry = dep_index.get(target) + if entry is None: + return None + with open(entry, encoding="utf-8") as fh: + doc = parse_isc(fh.read(), entry, resolve_dependency=resolver) + cache[target] = doc + return doc + + with open(target["path"], encoding="utf-8") as fh: + source = fh.read() + # warm the compiled grammar + cache first, profile the top-level file alone + one_pass(files, dep_index) + pr = cProfile.Profile() + pr.enable() + parse_isc(source, target["path"], resolve_dependency=resolver) + pr.disable() + stats = pstats.Stats(pr) + stats.sort_stats("cumulative") + print(f"=== cProfile {target['rel']} ({len(source)} chars) ===") + stats.print_stats(28) + stats.sort_stats("tottime") + print("=== by tottime ===") + stats.print_stats(28) + return 0 + + times = [one_pass(files, dep_index) for _ in range(args.repeat)] + for i, t in enumerate(times): + print(f"pass {i + 1}: {t:.2f} s") + print(f"corpus files: {len(files)}; median: {statistics.median(times):.2f} s") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/interscript-py/src/interscript/grammar/isc_parser.py b/interscript-py/src/interscript/grammar/isc_parser.py index 53030ea0..a1cb2e3a 100644 --- a/interscript-py/src/interscript/grammar/isc_parser.py +++ b/interscript-py/src/interscript/grammar/isc_parser.py @@ -35,6 +35,12 @@ # them with the tree in walk order. _SPAN_CAPTURES = frozenset({"from", "to"}) +# The rules whose failures carry diagnostic state the error renderer +# below reads: "double_quoted" (Unterminated string) and "item_atom" +# (keyword-in-item-position). The interpreter keeps per-failure +# position tables for exactly these rules. +_DIAG_RULES = frozenset({"double_quoted", "item_atom"}) + # extract [A-Za-z0-9_]* at a position (the hand parser's peek_word) _WORD_RE = re.compile(r"[A-Za-z0-9_]*") @@ -94,7 +100,7 @@ def parse_isc(source, filename=None, resolve_dependency=None): on syntax errors. """ c = compile_grammar() - interp = PegInterpreter(c.rules, source, span_names=_SPAN_CAPTURES) + interp = PegInterpreter(c.rules, source, span_names=_SPAN_CAPTURES, diag_rules=_DIAG_RULES) try: tree, _end = interp.parse_root(c.entry) except PegError as e: diff --git a/interscript-py/src/interscript/grammar/peg.py b/interscript-py/src/interscript/grammar/peg.py index 0fd95f87..8bda52ff 100644 --- a/interscript-py/src/interscript/grammar/peg.py +++ b/interscript-py/src/interscript/grammar/peg.py @@ -1,5 +1,5 @@ # Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). -"""PEG interpreter for grammar tables produced by parg_reader.py. +"""Closure-compiled PEG engine for grammar tables produced by parg_reader.py. Reproduces the parsanol (parslet-lineage) execution and tree-building semantics exactly (mirroring interscript-ts's src/isc/grammar/peg.ts): @@ -28,29 +28,84 @@ tree (closing quotes/brackets are not captured), but the W2 override key is the byte-exact source text of the from atom — so the interpreter reports the exact match spans and the action layer - pairs them with the tree in walk order. Memoized rule applications - store and replay their span delta, so memoization stays sound. + pairs them with the tree in walk order. Span bookkeeping is + confined to the rules whose fixpoint analysis says they can reach + a span capture; the from/to captures themselves are appended in + the same walk order the flattener sees. - FAILURE DIAGNOSTICS: the interpreter tracks the furthest position - reached, the rule names that failed there, and every rule name that - failed anywhere, so isc_parser.py can render the two hand-parser - error messages the suite pins ("Unterminated string", keyword-in- - item-position) from real parse state instead of string matching. + reached and, for the rules isc_parser.py names as `diag_rules`, + every failure position — the raw state the two hand-parser error + messages ("Unterminated string", keyword-in-item-position) are + rendered from. Failures of rules outside `diag_rules` update only + the furthest-position counter. + +WO18 execution model — each rule table is compiled ONCE into a chain of +Python closures (no per-input node dispatch). The grammar semantics are +unchanged; what is compiled away is interpreter overhead: + + - FINAL VALUES: every sequence/repetition closure folds its parts + exactly the way the flattener (CanFlatten) would fold the tagged + list the tree-driven interpreter used to build — the intermediate + tag list and the flatten re-walk are gone. A small fixed set of + rules (`raw_rules`) keeps tagged values: rules referenced from a + named position whose value can be an empty repetition/maybe (the + named fold yields []/None there), where shape-sensitive action + code observes the difference. + + - span bookkeeping compiled out of every subtree whose fixpoint + analysis proves it cannot reach a span capture. + + - failure diagnostics recorded lazily: one farthest-position compare + per failed rule application, plus a (pos, rule) log entry only for + `diag_rules`; the tables isc_parser.py reads are rebuilt on the + error path only. + + - NO packrat memoization: the corpus revisits a (rule, position) + pair ~3% of the time, so the memo tables were pure overhead (and + the replaced hand parser had none). Re-evaluation is + observationally identical — same call order, equal values, same + span tuples, same farthest maxima, and the same (position, rule) + failure log after the per-name sequential dedup. + + - scanner fusion: a contiguous run of Lit/CharClass atoms (merged + char-class alternatives, char-class repetitions, complement scans + for `!guard`-guarded character repetitions, and rule references + that resolve to such atoms) compiles into ONE scanner closure that + matches the whole run and returns the same merged Slice the + flattener would produce; a named capture over such a run produces + the capture hash directly. Repetition bodies that start with + one-char atom alternatives fold the whole atom run into one Slice + per run (hybrid scan); the flattener joins consecutive string-like + items wholesale, so run granularity is invisible. + + - literal alternatives compile to a first-character bucket table + (ordered choice preserved within each bucket), and char-class + alternatives merge into one range table (every branch yields the + identical one-char Slice). + +Recursive rules (a rule whose reference graph reaches itself — the ISC +grammar has exactly one) bind their body through a level of indirection; +every other rule body binds directly, in dependency order (rule bodies +may reference rules declared later in the table). """ from __future__ import annotations -from dataclasses import dataclass - from .parg_reader import Alt, Cap, CharClass, Lit, Pred, Ref, Rep, Seq -@dataclass(frozen=True, slots=True) class Slice: """A matched span of input (parsanol Slice: offset + content).""" - offset: int - content: str + __slots__ = ("offset", "content") + + def __init__(self, offset: int, content: str) -> None: + self.offset = offset + self.content = content + + def __repr__(self) -> str: + return f"Slice(offset={self.offset!r}, content={self.content!r})" class PegError(Exception): @@ -61,6 +116,12 @@ def __init__(self, message: str, at: int) -> None: self.at = at +# The shared failure result. Its third slot is never read on failure +# (every consumer branches on result[0] first), so one tuple serves all +# failing applications and a failing node allocates nothing. +_FAIL = (False, None, -1) + + # --------------------------------------------------------------------------- # Flattening — direct port of Parsanol::CanFlatten. # --------------------------------------------------------------------------- @@ -192,269 +253,1198 @@ def _flatten_repetition(lst: list, named: bool) -> object: # --------------------------------------------------------------------------- -# Interpreter +# Compilation — grammar table -> closure chain (built once per table). # --------------------------------------------------------------------------- +# Scanner segment encodings: +# ("l", s, n) literal, consumes n chars +# ("c", ranges, charset|None) one-char class (charset: frozenset fast path) +# ("r", body_segs, mn, mx) greedy repetition of a contiguous atom run -def _has_zero_named_charclass_rep(rules: dict) -> bool: - """Shape guard for the Rep(CharClass) scan fast path. The fast path - returns the repetition's FLATTENED value directly (a joined Slice, or - "" when zero iterations), which is identical to the slow path's - flatten EXCEPT for a zero-capable repetition sitting DIRECTLY under a - named capture (flatten yields [] there, not ""). No such shape exists - in the canonical ISC grammar; if a future grammar edit introduces - one, the guard disables the fast path for the whole table instead of - silently changing tree shapes.""" - def walk(node: object) -> bool: - if isinstance(node, Cap): - inner = node.a - if ( - isinstance(inner, Rep) - and not inner.maybe - and inner.min == 0 - and isinstance(inner.a, CharClass) - ): - return True - return walk(inner) - if isinstance(node, (Seq, Alt)): - return any(walk(child) for child in node.a) - if isinstance(node, Pred): - return walk(node.a) - return False +def _merge_ranges(ranges) -> tuple: + """Sort + coalesce a char-class's ranges (membership-preserving).""" + out: list[tuple[int, int]] = [] + for lo, hi in sorted(ranges): + if out and lo <= out[-1][1] + 1: + if hi > out[-1][1]: + out[-1] = (out[-1][0], hi) + else: + out.append((lo, hi)) + return tuple(out) - return any(walk(node) for node in rules.values()) +def _charset_for(ranges) -> frozenset | None: + """A frozenset fast path for small multi-range classes.""" + if len(ranges) < 3: + return None + if sum(hi - lo + 1 for lo, hi in ranges) > 1024: + return None + return frozenset(chr(u) for lo, hi in ranges for u in range(lo, hi + 1)) -class PegInterpreter: - MAX_DEPTH = 200 - def __init__( - self, - rules: dict, - src: str, - span_names: frozenset[str] = frozenset(), - ) -> None: - self._rules = rules - self._src = src - self._span_names = span_names - # Per-rule memo tables (rule order assigned lazily; pos -> result - # tuple). Memo values: (ok, value, end_pos, spans_delta, fail_name). - self._rule_ix: dict[int, int] = {} - self._memo: list[dict[int, tuple]] = [] - self.farthest = 0 - self.depth = 0 - # (name, start, end) of successful span-tracked captures, parse order. - self.spans: list[tuple[str, int, int]] = [] - # Rule names that failed AT the furthest position, and everywhere; - # plus every failure position per rule name (bounded by the input - # size) — the raw state the hand-parity error diagnostics read. - self.failed_at_farthest: set[str] = set() - self.failed_rules: set[str] = set() - self.failed_positions: dict[str, list[int]] = {} - # Rep(CharClass) scan fast path (see the Rep branch), disabled by - # the shape guard when a tree-shape-changing pattern exists. - self._fast_charclass_rep = not _has_zero_named_charclass_rep(rules) +def _ranges_difference(ranges, sub) -> tuple: + """ranges minus the (sorted, coalesced) sub ranges — membership-wise.""" + out: list[tuple[int, int]] = [] + subs = sorted(sub) + for lo, hi in sorted(ranges): + cur = lo + for slo, shi in subs: + if shi < cur or slo > hi: + continue + if slo > cur: + out.append((cur, min(slo - 1, hi))) + cur = max(cur, shi + 1) + if cur > hi: + break + if cur <= hi: + out.append((cur, hi)) + return tuple(out) - # -- entry ------------------------------------------------------------ - def parse_root(self, root: str, start: int = 0) -> tuple[object, int]: - """Parse from `start`, requiring the ROOT rule plus full input - consumption. Returns (flattened capture tree, end position).""" - r = self._apply_rule(root, start) - ok, value, pos = r[0], r[1], r[2] - if not ok or pos != len(self._src): - at = pos if ok else self.farthest - raise PegError(self._describe_failure(root, at), at) - return flatten(value), pos +class _Compiled: + """The compiled form of one rule table (shared across parses).""" - def _describe_failure(self, root: str, at: int) -> str: - line = self._src.count("\n", 0, at) + 1 - col = at - self._src.rfind("\n", 0, at) - return f"Failed to match {root} at line {line} char {col}" + __slots__ = ("entries", "ix_of", "spanning", "raw_rules") + + def __init__(self, entries, ix_of, spanning, raw_rules) -> None: + self.entries = entries # rule ix -> entry closure + self.ix_of = ix_of # rule name -> ix + self.spanning = spanning # rule ix -> bool (subtree reaches a span capture) + self.raw_rules = raw_rules # frozenset: rules kept in tagged-value mode - # -- bookkeeping -------------------------------------------------------- - def _note(self, pos: int) -> None: - self.farthest = max(self.farthest, pos) +def _collect_refs(node, acc): + t = type(node) + if t is Ref: + acc.append(node.name) + elif t is Seq or t is Alt: + for ch in node.a: + _collect_refs(ch, acc) + elif t is Rep or t is Pred or t is Cap: + _collect_refs(node.a, acc) + return acc - def _note_failure(self, pos: int, name: str) -> None: - self.failed_rules.add(name) - positions = self.failed_positions.setdefault(name, []) - if not positions or positions[-1] != pos: - positions.append(pos) - if pos > self.farthest: - self.farthest = pos - self.failed_at_farthest.clear() - if pos == self.farthest: - self.failed_at_farthest.add(name) - # -- rules ---------------------------------------------------------------- +def _compile_rules(rules: dict, span_names: frozenset, diag_rules: frozenset) -> _Compiled: + names = list(rules) + ix_of = {n: i for i, n in enumerate(names)} - def _apply_rule(self, name: str, pos: int) -> tuple: - ix = self._rule_ix.get(name) - if ix is None: - ix = self._rule_ix[name] = len(self._memo) - self._memo.append({}) - table = self._memo[ix] - hit = table.get(pos) - if hit is not None: - ok, value, r_pos, spans_delta, fail_name = hit - if ok: - if spans_delta: - self.spans.extend(spans_delta) + # Unknown rule references fail at compile time (the interpreter used + # to raise PegError on first parse; the canonical grammar has none). + for name in names: + for ref in _collect_refs(rules[name], []): + if ref not in ix_of: + raise PegError(f"unknown rule {ref!r}", 0) + + # -- single-reference rule inlining (AST level) ----------------------- + # A rule referenced exactly once is a pure passthrough at its ref + # site: the rule boundary adds only an entry call, a depth-count + # increment and a failure-log entry. Inline its body into the ref + # site. Diagnostics rules (their failure log feeds error rendering) + # and the entry rule stay; cycles cannot form (substitution only + # removes references). + from collections import Counter + + def subst(node, repl): + t = type(node) + if t is Ref: + return repl.get(node.name, node) + if t is Seq: + return Seq(tuple(subst(ch, repl) for ch in node.a)) + if t is Alt: + return Alt(tuple(subst(ch, repl) for ch in node.a)) + if t is Rep: + return Rep(subst(node.a, repl), node.min, node.max, node.maybe) + if t is Pred: + return Pred(node.positive, subst(node.a, repl)) + if t is Cap: + return Cap(node.name, subst(node.a, repl)) + return node # Lit / CharClass + + counts = Counter() + for name in names: + counts.update(_collect_refs(rules[name], [])) + while True: + target = next( + ( + n + for n, k in counts.items() + if k == 1 and n not in diag_rules and n in ix_of + ), + None, + ) + if target is None: + break + holder = next(n for n in names if target in _collect_refs(rules[n], [])) + rules[holder] = subst(rules[holder], {target: rules[target]}) + counts = Counter() + for name in names: + counts.update(_collect_refs(rules[name], [])) + + # -- span-reach fixpoint: which rules can record a span capture ------ + def node_reaches_span(node) -> bool: + t = type(node) + if t is Cap: + return node.name in span_names or node_reaches_span(node.a) + if t is Seq or t is Alt: + for ch in node.a: + if node_reaches_span(ch): + return True + return False + if t is Rep or t is Pred: + return node_reaches_span(node.a) + return False # Lit / CharClass / Ref (refs via the fixpoint) + + spanning = [False] * len(names) + changed = True + while changed: + changed = False + for i, name in enumerate(names): + v = node_reaches_span(rules[name]) or any( + spanning[ix_of[r]] for r in _collect_refs(rules[name], []) + ) + if spanning[i] != v: + spanning[i] = v + changed = True + + # -- atom rules: bodies that compile to a single contiguous matcher -- + # (Ref sites inside scanners inline these; other sites call normally.) + atom_rule: dict[str, bool] = {} + for name in names: + body = rules[name] + t = type(body) + atom_rule[name] = ( + t is Lit + or t is CharClass + or (t is Alt and bool(body.a) and all(type(ch) is CharClass for ch in body.a)) + ) + + def _min_chars(segs) -> int: + total = 0 + for s in segs: + if s[0] == "l": + total += s[2] + elif s[0] == "c": + total += 1 else: - self._note_failure(pos, fail_name) - return (ok, value, r_pos) - node = self._rules.get(name) - if node is None: - raise PegError(f"unknown rule {name!r}", pos) - self.depth += 1 - if self.depth > self.MAX_DEPTH: - self.depth -= 1 - raise PegError(f"recursion limit exceeded in rule {name!r}", pos) - mark = len(self.spans) - r = self._apply(node, pos, False) - self.depth -= 1 - if r[0]: - delta = tuple(self.spans[mark:]) if len(self.spans) != mark else () - memo = (True, r[1], r[2], delta, "") - else: - self._note_failure(pos, name) - del self.spans[mark:] - memo = (False, None, pos, (), name) - table[pos] = memo - return r + total += s[2] * _min_chars(s[1]) + if total > 0: + return total # exact enough for the consumes-at-least-one guard + return total + + def seg_for(node): + """Scanner segments for a contiguous atom run, or None. + + Returns a single segment tuple for atom-like nodes or a list of + segments for sequences; every segment consumes at least one + character except repetitions, whose own minimum is carried in + the segment. + """ + t = type(node) + if t is Lit: + s = node.s + return ("l", s, len(s)) + if t is CharClass: + ranges = _merge_ranges(node.ranges) + return ("c", ranges, _charset_for(ranges)) + if t is Alt and node.a and all(type(ch) is CharClass for ch in node.a): + ranges = _merge_ranges([r for ch in node.a for r in ch.ranges]) + return ("c", ranges, _charset_for(ranges)) + if t is Alt and node.a and all(type(ch) is Lit and len(ch.s) == 1 for ch in node.a): + ranges = _merge_ranges([(ord(ch.s), ord(ch.s)) for ch in node.a]) + return ("c", ranges, _charset_for(ranges)) + if t is Ref: + if atom_rule.get(node.name): + return seg_for(rules[node.name]) + return None + if t is Seq: + segs = [] + for ch in node.a: + s = seg_for(ch) + if s is None: + return None + segs.append(s) + return segs if segs else None + if t is Rep and not node.maybe: + body = seg_for(node.a) + if body is None: + return None + body_segs = body if isinstance(body, list) else [body] + if _min_chars(body_segs) < 1: + return None # an empty-matchable body would loop forever + return ("r", body_segs, node.min, node.max) + return None + + # -- recursive rules (reference-graph reachability) -------------------- + direct_refs = {name: set(_collect_refs(rules[name], [])) for name in names} - # -- nodes ---------------------------------------------------------------- + def reach_set(name) -> set: + seen: set = set() + stack = [name] + while stack: + for ref in direct_refs[stack.pop()]: + if ref not in seen: + seen.add(ref) + stack.append(ref) + return seen + + cyclic = [name for name in names if name in reach_set(name)] + + # -- closures --------------------------------------------------------- + + def c_lit(s, n): + def lit(pos, I): + if I.src.startswith(s, pos): + end = pos + n + if end > I.farthest: + I.farthest = end + return (True, Slice(pos, s), end) + return _FAIL + + return lit + + def c_cc(ranges, charset): + def cc(pos, I): + src = I.src + if pos >= I.n: + return _FAIL + c = src[pos] + u = ord(c) + ok = charset is not None and u in charset + if not ok: + for lo, hi in ranges: + if lo <= u <= hi: + ok = True + break + if not ok: + return _FAIL + end = pos + 1 + if end > I.farthest: + I.farthest = end + return (True, Slice(pos, c), end) - def _apply(self, node: object, pos: int, consume_all: bool) -> tuple: - # Dispatch ordered by measured hotness (Seq/Lit/CharClass dominate - # the identifier/whitespace rules). - kind = type(node) - if kind is Seq: + return cc + + def c_alt_lits(buckets): + get = buckets.get + + def alt_lit(pos, I): + src = I.src + if pos >= I.n: + return _FAIL + lits = get(src[pos]) + if lits is not None: + for s, n in lits: + if src.startswith(s, pos): + end = pos + n + if end > I.farthest: + I.farthest = end + return (True, Slice(pos, s), end) + return _FAIL + + return alt_lit + + def c_alt(children): + def alt(pos, I): + for f in children: + r = f(pos, I) + if r[0]: + return r + return _FAIL + + return alt + + def c_alt_span(children): + def alt(pos, I): + spans = I.spans + for f in children: + mark = len(spans) + r = f(pos, I) + if r[0]: + return r + if len(spans) != mark: + del spans[mark:] + return _FAIL + + return alt + + def c_seq(children): + def seq(pos, I): cur = pos - parts: list = ["seq"] - children = node.a # type: ignore[union-attr] - mark = len(self.spans) - n = len(children) - for i, child in enumerate(children): - r = self._apply(child, cur, consume_all and i == n - 1) + parts = ["seq"] + for f in children: + r = f(cur, I) if not r[0]: - if len(self.spans) != mark: - del self.spans[mark:] - return (False, None, pos) + return _FAIL parts.append(r[1]) cur = r[2] return (True, parts, cur) - if kind is Lit: - s = node.s # type: ignore[union-attr] - src = self._src - if src.startswith(s, pos): - end = pos + len(s) - self._note(end) - return (True, Slice(pos, s), end) - return (False, None, pos) - if kind is CharClass: - # Character classes match ONE code point (Python strings are - # sequences of code points; the ts interpreter matches UTF-16 - # units and relies on textual repetitions rejoining surrogate - # halves — same tree content either way). Every class in the - # ISC grammar is BMP except the catch-all `chr` (%x00-10FFFF). - src = self._src - if pos >= len(src): - return (False, None, pos) - unit = ord(src[pos]) - for lo, hi in node.ranges: # type: ignore[union-attr] - if lo <= unit <= hi: - self._note(pos + 1) - return (True, Slice(pos, src[pos]), pos + 1) - return (False, None, pos) - if kind is Ref: - return self._apply_rule(node.name, pos) # type: ignore[union-attr] - if kind is Rep: - mn = node.min # type: ignore[union-attr] - mx = node.max # type: ignore[union-attr] - body = node.a # type: ignore[union-attr] - if ( - self._fast_charclass_rep - and not node.maybe # type: ignore[union-attr] - and type(body) is CharClass - ): - # Scan fast path: a pure char-class repetition folds to the - # same flattened value the slow path produces (a joined - # Slice, or "" for zero iterations). Shape-guarded — see - # _has_zero_named_charclass_rep. - src = self._src - ranges = body.ranges - n = len(src) - cur = pos + + return seq + + def c_seq_span(children): + def seq(pos, I): + spans = I.spans + cur = pos + parts = ["seq"] + mark = len(spans) + for f in children: + r = f(cur, I) + if not r[0]: + if len(spans) != mark: + del spans[mark:] + return _FAIL + parts.append(r[1]) + cur = r[2] + return (True, parts, cur) + + return seq + + def c_rep(body, mn, mx, maybe): + tag = "maybe" if maybe else "rep" + + def rep(pos, I): + items = [tag] + cur = pos + count = 0 + while mx is None or count < mx: + r = body(cur, I) + if not r[0]: + break + items.append(r[1]) + cur = r[2] + count += 1 + if count < mn: + if cur > I.farthest: + I.farthest = cur + return _FAIL + return (True, items, cur) + + return rep + + def c_rep_span(body, mn, mx, maybe): + tag = "maybe" if maybe else "rep" + + def rep(pos, I): + spans = I.spans + items = [tag] + cur = pos + count = 0 + mark = len(spans) + while mx is None or count < mx: + mark_i = len(spans) + r = body(cur, I) + if not r[0]: + if len(spans) != mark_i: + del spans[mark_i:] + break + items.append(r[1]) + cur = r[2] + count += 1 + if count < mn: + if cur > I.farthest: + I.farthest = cur + if len(spans) != mark: + del spans[mark:] + return _FAIL + return (True, items, cur) + + return rep + + def c_rep_scan(ranges, charset, mn, mx, named=False, raw=False): + # Pure char-class repetition: folds to the same value the generic + # path's flatten produces (a joined Slice; an empty run folds to + # "" — or, in a named position, [] — exactly what CanFlatten + # yields for the tagged repetition at that position). Direct + # named-capture bodies never take this path — they compile to + # capture scanners, which carry the same fold. In raw (tagged) + # rules the empty run returns the TAGGED list instead: a raw + # rule's consumers may flatten it either way, and shape-sensitive + # action code (actions.value_start) distinguishes [] from "". + def rep(pos, I): + src = I.src + n = I.n + cur = pos + if charset is not None: + cs = charset + while cur < n and src[cur] in cs: + cur += 1 + else: while cur < n: - unit = ord(src[cur]) - matched = False + u = ord(src[cur]) + ok = False for lo, hi in ranges: - if lo <= unit <= hi: - matched = True + if lo <= u <= hi: + ok = True break - if not matched: + if not ok: break cur += 1 - count = cur - pos - if count < mn: - self._note(cur) - return (False, None, pos) - self._note(cur) - if count == 0: - return (True, "", cur) + if mx is not None and cur - pos > mx: + cur = pos + mx + count = cur - pos + if cur > I.farthest: + I.farthest = cur + if count < mn: + return _FAIL + if count == 0: + if raw: + return (True, ["rep"], cur) + return (True, [] if named else "", cur) + return (True, Slice(pos, src[pos:cur]), cur) + + return rep + + def c_pred(body, positive): + def pred(pos, I): + r = body(pos, I) + if r[0] is not positive: + return _FAIL + return (True, None, pos) + + return pred + + def c_rep_hybrid(ranges, charset, generics, mn, mx, maybe, final_named): + # Repetition whose body is an alternation with one-char atom + # branches first (merged into `ranges`) followed by generic + # branches (`generics`: (closure, is_raw) pairs): whole unit runs + # scan in one loop (one Slice per run), generic branches apply + # per iteration. The item sequence is value-equivalent to the + # per-iteration original: the flattener joins consecutive + # string-like items wholesale, so a run Slice and per-char + # Slices flatten identically (same join order, same offsets), + # and hash/list classification never inspects them. + # final_named is None in tagged (raw) mode — the tagged items + # list is returned; otherwise it is the named flag for the + # eager flatten (final mode). + def rep(pos, I): + src = I.src + n = I.n + if final_named is None: + items = ["maybe" if maybe else "rep"] + else: + items = [] + cur = pos + count = 0 + while mx is None or count < mx: + run_start = cur + if charset is not None: + cs = charset + while cur < n and src[cur] in cs: + cur += 1 + else: + while cur < n: + u = ord(src[cur]) + ok = False + for lo, hi in ranges: + if lo <= u <= hi: + ok = True + break + if not ok: + break + cur += 1 + if cur > run_start: + if mx is not None and cur - run_start > mx - count: + cur = run_start + (mx - count) + if cur > I.farthest: + I.farthest = cur + items.append(Slice(run_start, src[run_start:cur])) + count += cur - run_start + if mx is not None and count >= mx: + break + continue + matched = False + for f, is_raw in generics: + r = f(cur, I) + if r[0]: + if is_raw and final_named is not None: + items.append(flatten(r[1], False)) + else: + items.append(r[1]) + cur = r[2] + count += 1 + matched = True + break + if not matched: + break + if count < mn: + if cur > I.farthest: + I.farthest = cur + return _FAIL + if final_named is None: + return (True, items, cur) + return (True, _flatten_repetition(items, final_named), cur) + + return rep + + def c_maybe_wrap(ranges, charset, generics): + # `[x]` over a rule whose body is a run scanner / hybrid scanner + # (`[ws]`): one application of the inner scanner, folded the way + # the flattener folds it — every item is string-like (unit runs + # are Slices; the guard requires it for the generic branches), so + # the fold is one joined Slice from the start position, or "" for + # zero width. The entry call, the item list and the fold all + # collapse into this one closure. + def rep(pos, I): + src = I.src + n = I.n + cur = pos + while True: + run_start = cur + if charset is not None: + cs = charset + while cur < n and src[cur] in cs: + cur += 1 + else: + while cur < n: + u = ord(src[cur]) + ok = False + for lo, hi in ranges: + if lo <= u <= hi: + ok = True + break + if not ok: + break + cur += 1 + if cur > run_start: + continue + matched = False + for f in generics: + r = f(cur, I) + if r[0]: + cur = r[2] + matched = True + break + if not matched: + break + if cur > pos: + if cur > I.farthest: + I.farthest = cur return (True, Slice(pos, src[pos:cur]), cur) - tag = "maybe" if node.maybe else "rep" # type: ignore[union-attr] - items: list = [tag] + return (True, "", pos) + + return rep + + def c_seq_final(children): + # children: (closure, is_raw) pairs. The sequence's value is the + # CanFlatten fold over its flattened parts — computed here once, + # instead of building the tagged list and re-walking it later. + def seq(pos, I): + cur = pos + parts: list = [] + for f, is_raw in children: + r = f(cur, I) + if not r[0]: + return _FAIL + if is_raw: + parts.append(flatten(r[1], False)) + else: + parts.append(r[1]) + cur = r[2] + return (True, _flatten_sequence(parts), cur) + + return seq + + def c_rep_final(body, body_raw, mn, mx, maybe, named): + # The repetition's value is the CanFlatten fold over its + # flattened items (named flag: direct capture bodies fold an + # empty repetition to [] instead of ""). Same computation the + # flattener performs on the tagged list, done here once. An + # optional (`[x]`, maybe) folds like the flattener's maybe + # branch: the single flattened item, or "" when it matched + # nothing — never a list. + def rep(pos, I): + parts: list = [] cur = pos count = 0 - mark = len(self.spans) while mx is None or count < mx: - mark_i = len(self.spans) - r = self._apply(body, cur, False) + r = body(cur, I) if not r[0]: - if len(self.spans) != mark_i: - del self.spans[mark_i:] break - items.append(r[1]) + if body_raw: + parts.append(flatten(r[1], False)) + else: + parts.append(r[1]) cur = r[2] count += 1 if count < mn: - self._note(cur) - if len(self.spans) != mark: - del self.spans[mark:] - return (False, None, pos) - return (True, items, cur) - if kind is Alt: - for branch in node.a: # type: ignore[union-attr] - mark = len(self.spans) - r = self._apply(branch, pos, consume_all) - if r[0]: - return r - if len(self.spans) != mark: - del self.spans[mark:] - return (False, None, pos) - if kind is Cap: - mark = len(self.spans) - r = self._apply(node.a, pos, consume_all) # type: ignore[union-attr] + if cur > I.farthest: + I.farthest = cur + return _FAIL + if maybe: + first = parts[0] if parts else None + if named or not (first is None or first == ""): + return (True, first, cur) + return (True, "", cur) + return (True, _flatten_repetition(parts, named), cur) + + return rep + + def c_cap(body, name, is_span_cap, body_raw): + # body_raw: the body's value is a tagged list (a raw-marked rule) + # that CanFlatten must resolve, with the named flag, before the + # capture hash is built. Final bodies are already flattened. + if is_span_cap: + def cap(pos, I): + spans = I.spans + mark = len(spans) + r = body(pos, I) + if not r[0]: + if len(spans) != mark: + del spans[mark:] + return _FAIL + spans.append((name, pos, r[2])) + if body_raw: + return (True, {name: flatten(r[1], True)}, r[2]) + return (True, {name: r[1]}, r[2]) + + return cap + + def cap(pos, I): + r = body(pos, I) if not r[0]: - if len(self.spans) != mark: - del self.spans[mark:] - return (False, None, pos) - if node.name in self._span_names: # type: ignore[union-attr] - self.spans.append((node.name, pos, r[2])) # type: ignore[union-attr] - return (True, {node.name: flatten(r[1], True)}, r[2]) # type: ignore[union-attr] - if kind is Pred: - mark = len(self.spans) - r = self._apply(node.a, pos, False) # type: ignore[union-attr] - if len(self.spans) != mark: - del self.spans[mark:] # zero-width: no capture survives a lookahead - matched = r[0] - if (node.positive and not matched) or (not node.positive and matched): # type: ignore[union-attr] - return (False, None, pos) - # Zero-width: position unchanged, value None. - return (True, None, pos) + return _FAIL + if body_raw: + return (True, {name: flatten(r[1], True)}, r[2]) + return (True, {name: r[1]}, r[2]) + + return cap + + def match_unit(seg, src, cur, n): + """Match one scanner unit; returns the new position or -1.""" + k = seg[0] + if k == "l": + if src.startswith(seg[1], cur): + return cur + seg[2] + return -1 + if cur >= n: + return -1 + u = ord(src[cur]) + ok = seg[2] is not None and u in seg[2] + if not ok: + for lo, hi in seg[1]: + if lo <= u <= hi: + ok = True + break + return cur + 1 if ok else -1 + + def match_segs(segs, src, cur, n): + """Match a contiguous atom run; returns the end or -1.""" + for seg in segs: + if seg[0] == "r": + body, mn, mx = seg[1], seg[2], seg[3] + start = cur + while mx is None or cur - start < mx: + p = match_segs(body, src, cur, n) + if p < 0: + break + cur = p + if cur - start < mn: + return -1 + else: + cur = match_unit(seg, src, cur, n) + if cur < 0: + return -1 + return cur + + def c_cap_scan(name, segs, is_span_cap, empty_value): + def cap_scan(pos, I): + src = I.src + n = I.n + cur = pos + hi = pos # high-water mark of matched positions (farthest state) + failed = False + for seg in segs: + if seg[0] == "r": + body, mn, mx = seg[1], seg[2], seg[3] + start = cur + while mx is None or cur - start < mx: + p = match_segs(body, src, cur, n) + if p < 0: + break + cur = p + if cur > hi: + hi = cur + if cur - start < mn: + failed = True + break + else: + p = match_unit(seg, src, cur, n) + if p < 0: + failed = True + break + cur = p + hi = cur + if failed: + if hi > I.farthest: + I.farthest = hi + return _FAIL + if hi > I.farthest: + I.farthest = hi + if is_span_cap: + I.spans.append((name, pos, cur)) + if cur > pos: + return (True, {name: Slice(pos, src[pos:cur])}, cur) + return (True, {name: empty_value}, cur) + + return cap_scan + + def c_entry(name, body, diag): + # No packrat memoization: the corpus revisits a (rule, position) + # pair ~3% of the time, so the memo tables were pure overhead — + # and the replaced hand parser (recursive descent) had none. + # Re-evaluation is observationally identical: the same calls run + # in the same order, producing equal values, the same span + # tuples (the span delta replay existed only for memo hits), the + # same farthest maxima (internal success notes only re-note + # positions already noted on first evaluation), and the same + # (position, rule) failure log entries (the per-name sequential + # dedup absorbs the replayed duplicates the memo used to add). + def entry(pos, I): + depth = I.depth + if depth >= 200: + raise PegError(f"recursion limit exceeded in rule {name!r}", pos) + I.depth = depth + 1 + r = body(pos, I) + I.depth = depth + if not r[0]: + if pos > I.farthest: + I.farthest = pos + if diag: + I.flog.append((pos, name)) + return r + + return entry + + # -- per-rule compilation ---------------------------------------------- + + entries: list = [None] * len(names) + + def node_spanning(node) -> bool: + """Whether this node's subtree can record a span capture (refs + resolve to the rule-level fixpoint flags).""" + t = type(node) + if t is Ref: + return spanning[ix_of[node.name]] + if t is Cap: + return node.name in span_names or node_spanning(node.a) + if t is Seq or t is Alt: + for ch in node.a: + if node_spanning(ch): + return True + return False + if t is Rep or t is Pred: + return node_spanning(node.a) + return False # Lit / CharClass + + def unit_class_of(node): + """The merged ranges of a one-char matcher, or None. + + A one-char matcher is a char class (direct or an all-class + alternation), a single-char literal (or all-single-char-literal + alternation), or a reference to a rule whose body compiles to + one. Every branch yields the identical one-char Slice, so the + merged class matches exactly the same set with the same value. + """ + t = type(node) + if t is CharClass: + return _merge_ranges(node.ranges) + if t is Lit and len(node.s) == 1: + return ((ord(node.s), ord(node.s)),) + if t is Alt and node.a: + if all(type(ch) is CharClass for ch in node.a): + return _merge_ranges([r for ch in node.a for r in ch.ranges]) + if all(type(ch) is Lit and len(ch.s) == 1 for ch in node.a): + return _merge_ranges([(ord(ch.s), ord(ch.s)) for ch in node.a]) + if t is Ref and atom_rule.get(node.name): + seg = seg_for(rules[node.name]) + if seg is None: + return None + if seg[0] == "c": + return seg[1] + if seg[0] == "l" and seg[2] == 1: + return ((ord(seg[1]), ord(seg[1])),) + return None + + def rep_scan_of(body, mn, mx, named, raw): + """Scan closures for pure char-class repetitions, or None. + + - Rep(class / all-class alt): the joined run. + - Rep(seq(!one-char-guard, class)): the class run minus the + guard set (the complement scan — raw_run / sq_body). The + original's per-iteration items are [None, one-char Slice] + sequence tags; the flattener resolves each to its slice and + joins, which is exactly the scan's run Slice. + """ + t = type(body) + if t is CharClass: + ranges = _merge_ranges(body.ranges) + return c_rep_scan(ranges, _charset_for(ranges), mn, mx, named, raw) + if t is Alt and body.a and all(type(ch) is CharClass for ch in body.a): + ranges = _merge_ranges([r for ch in body.a for r in ch.ranges]) + return c_rep_scan(ranges, _charset_for(ranges), mn, mx, named, raw) + if t is Seq and len(body.a) == 2: + first, second = body.a + if type(first) is Pred and not first.positive: + guard = unit_class_of(first.a) + inner = unit_class_of(second) + if guard is not None and inner is not None: + diff = _ranges_difference(inner, guard) + return c_rep_scan(diff, _charset_for(diff), mn, mx, named, raw) + return None + + # -- final-value mode: rules that must stay tagged -------------------- + # Every rule body compiles to its FINAL (already-flattened) value; + # the flatten pass over tagged lists disappears. A rule keeps its + # tagged value only when a consumer folds it with named=True: a + # rule referenced from a named position (a capture body or an + # alternation inside one) whose value can be an EMPTY tagged + # repetition/maybe — the named fold yields []/None there, which the + # rule's eager (named=False) fold cannot express. + def bare_empty(node, seen): + t = type(node) + if t is Rep: + return node.min == 0 + if t is Alt: + for ch in node.a: + if bare_empty(ch, seen): + return True + return False + if t is Ref: + if node.name in seen: + return False + return bare_empty(rules[node.name], seen | {node.name}) + return False # Lit / CharClass / Seq / Cap / Pred + + cap_named_refs: set = set() + + def mark_named(node, in_named): + t = type(node) + if t is Ref: + if in_named: + cap_named_refs.add(node.name) + return + if t is Cap: + mark_named(node.a, True) + return + if t is Alt: + for ch in node.a: + mark_named(ch, in_named) + return + if t is Seq: + for ch in node.a: + mark_named(ch, False) + return + if t is Rep or t is Pred: + mark_named(node.a, False) + + for name in names: + mark_named(rules[name], False) + + raw_rules = frozenset( + name for name in cap_named_refs if bare_empty(rules[name], frozenset()) + ) + + def stringlike_final(node, seen): + """Whether the node's FINAL value is always string-like (Slice + or str). None parts inside a sequence fold are skipped by the + fold, so they count there.""" + t = type(node) + if t is Lit or t is CharClass or t is Pred: + return True # Slice / Slice / None (skipped by the fold) + if t is Ref: + if node.name in seen: + return False + return stringlike_final(rules[node.name], seen | {node.name}) + if t is Seq or t is Alt: + for ch in node.a: + if not stringlike_final(ch, seen): + return False + return True + if t is Rep: + return not node.a or stringlike_final(node.a, seen) + return False # Cap (a hash) + + def _node_min_chars(node): + t = type(node) + if t is Lit: + return len(node.s) + if t is CharClass: + return 1 + if t is Seq: + return sum(_node_min_chars(ch) for ch in node.a) + if t is Alt and node.a: + return min(_node_min_chars(ch) for ch in node.a) + if t is Rep: + body = _node_min_chars(node.a) + return node.min * body if body else 0 + if t is Cap or t is Pred: + return 0 + if t is Ref: + return _node_min_chars(rules[node.name]) + return 0 + + def maybe_wrap_of(rule_name): + """A fused `[x]` over a rule whose body is a run/hybrid scanner + (`[ws]`): None when the shape does not apply. The wrap value is + the maybe-fold of one inner application — every item string-like, + hence one joined Slice or "".""" + rb = rules[rule_name] + if type(rb) is not Rep or rb.maybe or rb.min != 0 or rb.max is not None: + return None + inner = rb.a + if not node_spanning(rb): + scan = rep_scan_of(inner, rb.min, rb.max, False, False) + if scan is not None: + # one scan application; its zero-width value is "" — + # exactly the maybe fold of a zero-width application + return scan + if type(inner) is Alt and inner.a and not node_spanning(rb): + unit_ranges: list = [] + rest = list(inner.a) + while rest: + u = unit_class_of(rest[0]) + if u is None: + break + unit_ranges.extend(u) + rest.pop(0) + if unit_ranges and rest and all( + type(ch) is Ref + and stringlike_final(rules[ch.name], frozenset({ch.name})) + and _node_min_chars(rules[ch.name]) >= 1 + for ch in rest + ): + ranges = _merge_ranges(unit_ranges) + generics = tuple(build(ch, False, False) for ch in rest) + return c_maybe_wrap(ranges, _charset_for(ranges), generics) + return None + + def build(node, raw=False, named=False): + t = type(node) + if t is Lit: + return c_lit(node.s, len(node.s)) + if t is CharClass: + ranges = _merge_ranges(node.ranges) + return c_cc(ranges, _charset_for(ranges)) + if t is Alt: + branches = node.a + if branches and all(type(ch) is Lit for ch in branches): + if all(ch.s for ch in branches): # no empty literals + buckets: dict = {} + for ch in branches: + buckets.setdefault(ch.s[0], []).append((ch.s, len(ch.s))) + return c_alt_lits({k: tuple(v) for k, v in buckets.items()}) + if branches and all(type(ch) is CharClass for ch in branches): + ranges = _merge_ranges([r for ch in branches for r in ch.ranges]) + return c_cc(ranges, _charset_for(ranges)) + children = tuple(build(ch, raw, named) for ch in branches) + return c_alt(children) + if t is Seq: + if raw: + children = tuple(build(ch, True, False) for ch in node.a) + return c_seq(children) + children = tuple( + (build(ch, False, False), type(ch) is Ref and ch.name in raw_rules) + for ch in node.a + ) + return c_seq_final(children) + if t is Rep: + body = node.a + if not node.maybe: + scan = rep_scan_of(body, node.min, node.max, named, raw) + if scan is not None: + return scan + if ( + not raw + and node.maybe + and node.min == 0 + and node.max == 1 + and type(body) is Ref + ): + wrap = maybe_wrap_of(body.name) + if wrap is not None: + return wrap + if not node.maybe and type(body) is Alt and body.a and not node_spanning(node): + # hybrid scan: leading one-char atom branches merge into a + # unit run; the remaining branches stay per-iteration + unit_ranges: list = [] + rest = list(body.a) + while rest: + u = unit_class_of(rest[0]) + if u is None: + break + unit_ranges.extend(u) + rest.pop(0) + if unit_ranges and rest: + ranges = _merge_ranges(unit_ranges) + if raw: + generics = tuple((build(ch, True, named), False) for ch in rest) + return c_rep_hybrid( + ranges, _charset_for(ranges), generics, + node.min, node.max, node.maybe, None, + ) + generics = tuple( + (build(ch, False, named), type(ch) is Ref and ch.name in raw_rules) + for ch in rest + ) + return c_rep_hybrid( + ranges, _charset_for(ranges), generics, + node.min, node.max, node.maybe, named, + ) + if raw: + return c_rep(build(body, True, False), node.min, node.max, node.maybe) + body_raw = type(body) is Ref and body.name in raw_rules + return c_rep_final( + build(body, False, False), body_raw, node.min, node.max, node.maybe, named + ) + if t is Pred: + return c_pred(build(node.a, raw, False), node.positive) + if t is Cap: + segs = seg_for(node.a) + if segs is not None: + if isinstance(segs, list): + empty_value = "" # sequence body: an empty textual run folds to "" + else: + segs = [segs] + empty_value = [] # direct repetition body: named empty -> [] + return c_cap_scan(node.name, segs, node.name in span_names, empty_value) + body_node = node.a + span_cap = node.name in span_names + if raw: + return c_cap(build(body_node, True, True), node.name, span_cap, True) + body_raw = type(body_node) is Ref and body_node.name in raw_rules + # The capture body is a named position intrinsically: a direct + # repetition/alternation body folds with named=True (its eager + # fold must yield [] on an empty run), a sequence body does + # not (its children fold with named=False either way). + child_named = not body_raw and type(body_node) in (Rep, Alt) + return c_cap(build(body_node, False, child_named), node.name, span_cap, body_raw) + if t is Ref: + return entries[ix_of[node.name]] raise TypeError(f"BUG: unknown PegNode {node!r}") + + # Recursive rules get their entries first (refs bind to them), with + # the body threaded through a one-slot cell; acyclic rules then + # compile in dependency order (a rule body may reference rules + # declared later in the table), with direct body binding. + cells: dict[str, list] = {} + for name in cyclic: + ix = ix_of[name] + cell = [None] + cells[name] = cell + entries[ix] = c_entry( + name, (lambda p, I, _c=cell: _c[0](p, I)), name in diag_rules + ) + remaining = [name for name in names if entries[ix_of[name]] is None] + built = {name for name in cyclic} + while remaining: + progressed = False + still: list = [] + for name in remaining: + if all(d in built for d in direct_refs[name]): + body = build(rules[name], raw=name in raw_rules) + entries[ix_of[name]] = c_entry( + name, body, name in diag_rules + ) + built.add(name) + progressed = True + else: + still.append(name) + if not progressed: + raise RuntimeError(f"grammar build order unresolved for: {still!r}") + remaining = still + for name, cell in cells.items(): + cell[0] = build(rules[name], raw=name in raw_rules) + + return _Compiled(entries, ix_of, spanning, raw_rules) + + +# Per-rule-table compile cache: id -> (rules, span_names, diag_rules, +# compiled). The rules reference is kept alive so its id is never reused. +_COMPILE_CACHE: dict = {} + + +def _compiled_for(rules: dict, span_names: frozenset, diag_rules: frozenset) -> _Compiled: + key = id(rules) + hit = _COMPILE_CACHE.get(key) + if hit is not None and hit[0] is rules and hit[1] == span_names and hit[2] == diag_rules: + return hit[3] + compiled = _compile_rules(rules, span_names, diag_rules) + _COMPILE_CACHE[key] = (rules, span_names, diag_rules, compiled) + return compiled + + +# --------------------------------------------------------------------------- +# Interpreter — per-parse state over the compiled closures. +# --------------------------------------------------------------------------- + + +class PegInterpreter: + MAX_DEPTH = 200 + + def __init__( + self, + rules: dict, + src: str, + span_names: frozenset[str] = frozenset(), + diag_rules: frozenset[str] = frozenset(), + ) -> None: + compiled = _compiled_for(rules, span_names, diag_rules) + self._compiled = compiled + # Per-parse state, read by the compiled closures. Attribute names + # are part of the closure contract. + self.src = src + self.n = len(src) + self.farthest = 0 + self.depth = 0 + # (name, start, end) of successful span-tracked captures, parse order. + self.spans: list[tuple[str, int, int]] = [] + # Failure log: (pos, rule) for diag_rules failures, in parse order; + # rebuilt into the diagnostic tables on the error path only. + self.flog: list[tuple[int, str]] = [] + self._diag: tuple[set, dict] | None = None + + # -- entry ------------------------------------------------------------ + + def parse_root(self, root: str, start: int = 0) -> tuple[object, int]: + """Parse from `start`, requiring the ROOT rule plus full input + consumption. Returns (flattened capture tree, end position).""" + ix = self._compiled.ix_of.get(root) + if ix is None: + raise PegError(f"unknown rule {root!r}", start) + r = self._compiled.entries[ix](start, self) + if not r[0] or r[2] != self.n: + at = r[2] if r[0] else self.farthest + raise PegError(self._describe_failure(root, at), at) + value = r[1] if root not in self._compiled.raw_rules else flatten(r[1]) + return value, r[2] + + def _describe_failure(self, root: str, at: int) -> str: + line = self.src.count("\n", 0, at) + 1 + col = at - self.src.rfind("\n", 0, at) + return f"Failed to match {root} at line {line} char {col}" + + # -- failure diagnostics (error-path only) ------------------------------ + + def _build_diag(self) -> tuple[set, dict]: + failed_rules: set[str] = set() + failed_positions: dict[str, list[int]] = {} + for pos, name in self.flog: + failed_rules.add(name) + positions = failed_positions.setdefault(name, []) + if not positions or positions[-1] != pos: + positions.append(pos) + return failed_rules, failed_positions + + @property + def failed_rules(self) -> set: + if self._diag is None: + self._diag = self._build_diag() + return self._diag[0] + + @property + def failed_positions(self) -> dict: + if self._diag is None: + self._diag = self._build_diag() + return self._diag[1]