diff --git a/interscript-py/scripts/parse_parity_py.py b/interscript-py/scripts/parse_parity_py.py new file mode 100644 index 00000000..aced1c98 --- /dev/null +++ b/interscript-py/scripts/parse_parity_py.py @@ -0,0 +1,371 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""py-vs-py ISC parse parity driver — the swap gate for WO17 step 4b. + +Parses every corpus file (maps/*.isc + libs/*.isc of an interscript/maps +checkout) with TWO python ISC parsers and proves they agree: + + - HAND: the replaced hand-written recursive-descent parser, + materialized from a git revision of this repository (default: + HEAD — run this BEFORE committing the parser swap; pass + --hand-rev after). + - GENERATED: the grammar-driven parser shipped in + src/interscript/grammar/ (what interscript.isc.parse_isc is now). + +Every parsed document (dependencies resolved against the same corpus, +each side with its own parser) is projected onto the maps +scripts/parse-parity.mjs canonical schema (the py normalizer: constraint +kinds re-sorted into SubRule's fixed fields, `none` lowered to ""). +The driver reports TWO numbers: + + - canonical: identical under the canonical schema (the official + judge — the same projection the cross-runtime harness compares). + - raw: identical field-for-field INCLUDING from_source and the + written constraint order (strictest swap-safety bar; a superset + of the canonical comparison). + +Usage: + python scripts/parse_parity_py.py [--maps-root ] + [--hand-rev ] [--json ] [--only ] + [--verbose] + +Exit code 0 only when EVERY corpus file passes BOTH projections. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +PY_SRC = Path(__file__).resolve().parents[1] / "src" + +sys.path.insert(0, str(PY_SRC)) + +from interscript.isc import parse_isc as generated_parse_isc + +# The py normalizer's keyword tables (mirrored from maps +# scripts/parse-parity.mjs — identical to both parsers'). +PRIMITIVES = frozenset( + ["boundary", "line_start", "line_end", "word_boundary", "space", "non_boundary"] +) +FUNCTIONS = frozenset(["upcase", "downcase", "title_case", "reverse", "strip", "swapcase"]) +PY_CONSTRAINT_ORDER = ["before", "after", "not_before", "not_after"] + + +# --------------------------------------------------------------------------- +# Hand parser materialization — the replaced isc.py from a git revision +# --------------------------------------------------------------------------- + + +def load_hand_parser(repo_root: Path, rev: str): + """Materialize the hand isc.py at `rev` into a temp package (its + siblings _ir_types.py / errors.py are copied from the current tree — + they are identical across the swap) and import it.""" + def show(path: str) -> str: + out = subprocess.run( + ["git", "-C", str(repo_root), "show", f"{rev}:{path}"], + capture_output=True, + text=True, + check=True, + ) + return out.stdout + + isc_src = show("interscript-py/src/interscript/isc.py") + if "from .grammar.isc_parser import" in isc_src: + raise SystemExit( + f"parse_parity_py: rev {rev!r} already carries the generated parser — " + f"pass --hand-rev " + ) + tmp = Path(tempfile.mkdtemp(prefix="hand-isc-")) + pkg = tmp / "hand_isc_pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + (pkg / "isc.py").write_text(isc_src, encoding="utf-8") + for sibling in ("_ir_types.py", "errors.py"): + (pkg / sibling).write_text( + (PY_SRC / "interscript" / sibling).read_text(encoding="utf-8"), + encoding="utf-8", + ) + spec = importlib.util.spec_from_file_location( + "hand_isc_pkg.isc", pkg / "isc.py", + submodule_search_locations=[str(pkg)], + ) + module = importlib.util.module_from_spec(spec) + sys.modules["hand_isc_pkg"] = type(sys)("hand_isc_pkg") + sys.modules["hand_isc_pkg"].__path__ = [str(pkg)] + sys.modules["hand_isc_pkg.isc"] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + return module.parse_isc + + +# --------------------------------------------------------------------------- +# Canonical schema — port of maps scripts/parse-parity.mjs's py normalizer +# --------------------------------------------------------------------------- + + +def jsonable(x): + if dataclasses.is_dataclass(x) and not isinstance(x, type): + d = {"__kind__": type(x).__name__} + for f in dataclasses.fields(x): + d[f.name] = jsonable(getattr(x, f.name)) + return d + if isinstance(x, (list, tuple)): + return [jsonable(v) for v in x] + if isinstance(x, dict): + return {k: jsonable(v) for k, v in x.items()} + return x + + +def canon_item(it): + if not isinstance(it, dict): + return {"t": "shape", "raw": str(it)} + kind = it["__kind__"] + if kind == "StringItem": + return {"t": "s", "v": it["value"]} + if kind == "AliasItem": + if it["name"] in PRIMITIVES: + return {"t": "prim", "name": it["name"]} + if it["name"] in FUNCTIONS: + return {"t": "fn", "name": it["name"]} + return {"t": "alias", "name": it["name"]} + if kind == "CaptureGroupItem": + return {"t": "cap", "inner": canon_item(it["data"])} + if kind == "CaptureRefItem": + return {"t": "ref", "id": it["id"]} + if kind == "AnyItem": + return {"t": "any", "of": [canon_item(i) for i in it["of"]]} + if kind == "AnyCharClassItem": + if it["range"]: + return {"t": "range", "lo": it["range"][0], "hi": it["range"][1]} + if it["chars"]: + return {"t": "any", "of": [{"t": "s", "v": c} for c in it["chars"]]} + return {"t": "shape", "raw": "any_char_class with neither range nor chars"} + if kind == "GroupItem": + return {"t": "cat", "parts": [canon_item(i) for i in it["items"]]} + if kind == "RepeatItem": + if it["min"] == 0 and it["max"] == 1: + return {"t": "maybe", "inner": canon_item(it["item"])} + if it["min"] == 1 and it["max"] is None: + return {"t": "some", "inner": canon_item(it["item"])} + return {"t": "shape", "raw": f"repeat min={it['min']} max={it['max']}"} + if kind == "FuncallInline": + return {"t": "fn", "name": it["name"]} + if kind == "StageItem": + return {"t": "stage_ref", "name": it["name"]} + return {"t": "shape", "raw": json.dumps(it)} + + +def canon_rule(r): + return { + "from": canon_item(r["from_"]), + "to": canon_item(r["to"]), + "constraints": [ + {"kind": k, "item": canon_item(r[k])} for k in PY_CONSTRAINT_ORDER if r[k] is not None + ], + **({"when": list(r["when"])} if r["when"] else {}), + } + + +def canon_stage_item(it): + kind = it["__kind__"] + if kind == "ParsedRule": + return {"k": "rule", "rule": canon_rule(it["rule"]), "fromSource": it["from_source"]} + if kind == "ParallelBlock" or kind == "SequenceBlock": + return { + "k": "parallel" if kind == "ParallelBlock" else "sequence", + "rules": [canon_rule(r["rule"]) for r in it["rules"]], + } + if kind == "RunItem": + out = {"k": "run", "stage": it["stage"]} + if it["dependency"]: + out["dependency"] = it["dependency"] + return out + if kind == "FuncallRule": + if it["name"] == "separate": + sep = (it.get("kwargs") or {}).get("separator") + return {"k": "separate", **({"separator": {"t": "s", "v": sep}} if sep is not None else {})} + if it["name"] == "compose": + return {"k": "compose"} + if it["name"] == "decompose": + return {"k": "decompose"} + if it["name"] in FUNCTIONS: + return {"k": "string_case", "op": it["name"]} + return {"k": "funcall", "name": it["name"], "kwargs": dict(it.get("kwargs") or {})} + return {"k": "shape", "raw": json.dumps(it)} + + +def canon_doc(doc): + return { + "systemCode": doc.system_code, + "metadata": jsonable(doc.metadata) or {}, + "tests": [ + {"input": t.input, "expected": t.expected, **({"note": t.note} if t.note is not None else {})} + for t in doc.tests + ], + "dependencies": [ + {"target": d.target, **({"alias": d.alias_name} if d.alias_name is not None else {})} + for d in doc.dependencies + ], + "aliases": [{"name": a[0], "item": canon_item(a[1])} for a in doc.aliases], + **({"languages": list(doc.languages)} if doc.languages else {}), + "stages": [{"name": s.name, "body": [canon_stage_item(jsonable(i)) for i in s.items]} for s in doc.stages], + "rulesets": { + name: [canon_stage_item(jsonable(i)) for i in body] + for name, body in doc.rulesets.items() + }, + } + + +# --------------------------------------------------------------------------- +# Comparison with path reporting +# --------------------------------------------------------------------------- + + +def walk(a, b, path, section, divergences): + if isinstance(a, list) and isinstance(b, list): + if len(a) != len(b): + divergences.append({"kind": section, "path": path, "hand": f"len={len(a)}", "gen": f"len={len(b)}"}) + for i in range(min(len(a), len(b))): + walk(a[i], b[i], f"{path}[{i}]", section, divergences) + return + if not isinstance(a, dict) or not isinstance(b, dict): + if a != b: + divergences.append({"kind": section, "path": path, "hand": json.dumps(a, ensure_ascii=False)[:400], "gen": json.dumps(b, ensure_ascii=False)[:400]}) + return + for key in set(a) | set(b): + if key not in a or key not in b: + divergences.append({"kind": section, "path": f"{path}.{key}", "hand": json.dumps(a.get(key), ensure_ascii=False)[:400], "gen": json.dumps(b.get(key), ensure_ascii=False)[:400]}) + continue + walk(a[key], b[key], f"{path}.{key}", section, divergences) + + +def compare_docs(a, b, sections, divergences): + for section in sections: + if section not in a and section not in b: + continue + if section not in a or section not in b: + divergences.append({"kind": section, "path": section, "hand": json.dumps(a.get(section)), "gen": json.dumps(b.get(section))}) + continue + walk(a[section], b[section], section, section, divergences) + + +CANON_SECTIONS = ["systemCode", "metadata", "tests", "dependencies", "aliases", "languages", "stages", "rulesets"] + + +def parse_with(parse_fn, source, path, dep_index, cache): + def resolver(target): + if target in cache: + return cache[target] + entry = dep_index.get(target) + if entry is None: + return None + with open(entry, encoding="utf-8") as fh: + doc = parse_fn(fh.read(), entry, resolve_dependency=resolver) + cache[target] = doc + return doc + + with open(path, encoding="utf-8") as fh: + source = fh.read() + return parse_fn(source, path, resolve_dependency=resolver) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--maps-root", default=str(REPO_ROOT.parent / "maps")) + ap.add_argument("--hand-rev", default="HEAD") + ap.add_argument("--json", dest="json_out") + ap.add_argument("--only") + ap.add_argument("--verbose", action="store_true") + args = ap.parse_args() + + maps_root = Path(args.maps_root).resolve() + files = [] + dep_index = {} + for sub in ("maps", "libs"): + d = maps_root / sub + for f in sorted(d.iterdir()): + if f.suffix == ".isc": + files.append({"rel": f"{sub}/{f.name}", "path": f}) + dep_index[f.stem] = f + if args.only: + files = [f for f in files if args.only in f["rel"]] + if not files: + print(f"parse_parity_py: no corpus files under {maps_root}/maps and /libs", file=sys.stderr) + return 2 + + hand_parse = load_hand_parser(REPO_ROOT, args.hand_rev) + + hand_cache: dict = {} + gen_cache: dict = {} + + reports = [] + canon_pass = 0 + raw_pass = 0 + for f in files: + divergences_canon = [] + divergences_raw = [] + try: + hand_doc = parse_with(hand_parse, "", f["path"], dep_index, hand_cache) + except Exception as e: # noqa: BLE001 + hand_doc = e + try: + gen_doc = parse_with(generated_parse_isc, "", f["path"], dep_index, gen_cache) + except Exception as e: # noqa: BLE001 + gen_doc = e + + if isinstance(hand_doc, Exception) or isinstance(gen_doc, Exception): + divergences_canon.append({ + "kind": "error", + "path": "parse", + "hand": type(hand_doc).__name__ + ": " + str(hand_doc) if isinstance(hand_doc, Exception) else "ok", + "gen": type(gen_doc).__name__ + ": " + str(gen_doc) if isinstance(gen_doc, Exception) else "ok", + }) + status = "FAIL" + else: + compare_docs(canon_doc(hand_doc), canon_doc(gen_doc), CANON_SECTIONS, divergences_canon) + compare_docs( + jsonable(hand_doc), jsonable(gen_doc), + ["system_code", "metadata", "tests", "aliases", "stages", "dependencies", "languages", "rulesets"], + divergences_raw, + ) + canon_ok = not divergences_canon + raw_ok = not divergences_raw + canon_pass += canon_ok + raw_pass += raw_ok + status = "PASS" if (canon_ok and raw_ok) else "FAIL" + reports.append({ + "path": f["rel"], "status": status, + "canonical": "PASS" if canon_ok else divergences_canon, + "raw": "PASS" if raw_ok else divergences_raw, + }) + if status == "PASS": + print(f"{f['rel']} PASS") + else: + first = (divergences_canon or divergences_raw)[0] + print(f"{f['rel']} FAIL canon={len(divergences_canon)} raw={len(divergences_raw)}; first: {first['kind']} @ {first['path']}") + if args.verbose: + for d in (divergences_canon + divergences_raw)[:12]: + print(f" {d['kind']} @ {d['path']}\n hand: {d['hand']}\n gen: {d['gen']}") + + print( + f"parity: canonical {canon_pass}/{len(files)}; raw {raw_pass}/{len(files)} " + f"(hand rev {args.hand_rev})" + ) + if args.json_out: + Path(args.json_out).write_text(json.dumps({"files": reports, "handRev": args.hand_rev}, indent=2), encoding="utf-8") + return 0 if (canon_pass == len(files) and raw_pass == len(files)) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/interscript-py/scripts/sync_isc_grammar.py b/interscript-py/scripts/sync_isc_grammar.py new file mode 100644 index 00000000..a9cbe302 --- /dev/null +++ b/interscript-py/scripts/sync_isc_grammar.py @@ -0,0 +1,85 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Sync the vendored ISC grammar (src/interscript/grammar/grammar_source.py) +from a canonical interscript-ruby checkout: + + python scripts/sync_isc_grammar.py [path-to-interscript-ruby] + +Defaults to the checkout that contains this package (the interscript-py +subtree lives inside interscript-ruby, so the canonical +lib/interscript/isc/grammar/isc.parg sits two levels up). Rewrites the +embedded source and the sha256 pin together, so the pin test +(tests/test_grammar_source.py) always describes the text that ships. + +This mirrors interscript-ts's scripts/sync-isc-grammar.mjs; both vendored +copies must carry the same sha256 pin. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +DEFAULT_PARG = Path("lib/interscript/isc/grammar/isc.parg") +OUT_PATH = Path(__file__).resolve().parents[1] / "src" / "interscript" / "grammar" / "grammar_source.py" + +HEADER = '''\ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Vendored copy of the canonical ISC grammar: + + interscript-ruby lib/interscript/isc/grammar/isc.parg + +The .parg file is the single semantic source of the ISC syntax; this +module embeds it verbatim (sha256 {sha}) so the Python parser compiles +the same grammar text the Ruby runtime ships. Update procedure: + + python scripts/sync_isc_grammar.py + +The sync script rewrites this file and the sha256 pin; the pin test +(tests/test_grammar_source.py) fails when the embedded text and the +recorded hash disagree, so drift is never silent. + +This is the same file (same sha256 pin) that interscript-ts vendors via +its scripts/sync-isc-grammar.mjs — the two generated parsers must never +compile different grammar texts. +""" + +from __future__ import annotations + +# sha256 of the grammar text embedded below (pin for drift detection). +ISC_PARG_SHA256 = "{sha}" + +# The grammar text itself, byte-identical to the ruby checkout copy. +ISC_PARG_SOURCE = {source} +''' + + +def main() -> int: + argv = sys.argv[1:] + if argv: + root = Path(argv[0]).expanduser().resolve() + else: + # interscript-py/.. -> the interscript-ruby checkout that ships us. + root = Path(__file__).resolve().parents[2] + parg_path = root / DEFAULT_PARG + try: + src = parg_path.read_text(encoding="utf-8") + except OSError as exc: + print(f"sync_isc_grammar: cannot read {parg_path}: {exc}", file=sys.stderr) + print( + "sync_isc_grammar: pass the path to an interscript-ruby checkout as argv[1]", + file=sys.stderr, + ) + return 2 + + sha = hashlib.sha256(src.encode("utf-8")).hexdigest() + out = HEADER.format(sha=sha, source=json.dumps(src)) + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(out, encoding="utf-8") + print(f"sync_isc_grammar: wrote {OUT_PATH} (sha256 {sha})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/interscript-py/src/interscript/grammar/__init__.py b/interscript-py/src/interscript/grammar/__init__.py new file mode 100644 index 00000000..9b7dd210 --- /dev/null +++ b/interscript-py/src/interscript/grammar/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Grammar-driven ISC parser (generated from the canonical .parg). + +Modules: +- grammar_source — the vendored .parg text, sha256-pinned +- parg_reader — PARG lexer/parser (Parsanol::PARG subset) +- peg — table-driven PEG interpreter + parsanol flatten +- actions — capture tree -> IscDocument (hand-parser parity) +- isc_parser — compiled grammar cache + the parse_isc entry + +Public entry: interscript.isc.parse_isc (kept stable there). +""" diff --git a/interscript-py/src/interscript/grammar/actions.py b/interscript-py/src/interscript/grammar/actions.py new file mode 100644 index 00000000..3ba37f0f --- /dev/null +++ b/interscript-py/src/interscript/grammar/actions.py @@ -0,0 +1,616 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Action layer — turns the parslet-shaped capture tree produced by the +PEG interpreter (peg.py) into an IscDocument. + +Every action mirrors the replaced hand parser's (the former isc.py +_Parser) semantic action for the same construct, so the generated +parser is a drop-in replacement: identical documents for identical +source — including the W1/W2 ruleset expansion, which runs here through +isc._Expander (the same code the hand parser used). Where the hand +parser deviates from the grammar (each case is corpus-proven dormant by +the 434-file py-vs-py parity driver), the deviation is reproduced and +flagged with a DEVIATION comment — grammar fidelity loses to +swap-safety, on purpose, until the owners decide to align the runtimes. + +Mirrors interscript-ts's src/isc/grammar/actions.ts, with the py +runtime's document shapes (SubRule fixed constraint fields, FuncallRule +stage items, from_source override keys, parse-time use expansion). +""" + +from __future__ import annotations + +from typing import Any + +from .._ir_types import ( + AliasItem, + AnyCharClassItem, + AnyItem, + CaptureGroupItem, + CaptureRefItem, + FuncallRule, + GroupItem, + RepeatItem, + StringItem, +) +from ..isc import ( + IscDependency, + IscDocument, + IscStage, + IscTest, + ParallelBlock, + ParsedRule, + RawUse, + RunItem, + SequenceBlock, + _Expander, + _function_to_inline, + _make_sub_rule, +) +from .peg import Slice + +Tree = object +Hash = dict + + +def _hash(v: Tree) -> Hash: + if not isinstance(v, dict): + raise TypeError( + f"grammar/action mismatch: expected a capture hash, got {v!r}" + ) + return v + + +def _array(v: Tree) -> list: + if not isinstance(v, list): + raise TypeError(f"grammar/action mismatch: expected an array, got {v!r}") + return v + + +def _identifier(tree: Tree) -> str: + """Text of a captured identifier (a merged slice).""" + v = _hash(tree)["identifier"] + return v.content if isinstance(v, Slice) else v # type: ignore[return-value,union-attr] + + +def _dependency_name(tree: Tree) -> str: + """A `run map.a-b.stage.x` dependency name: the hyphen extension + captures `identifier *("-" identifier)`, so a hyphenated name arrives + as a list of identifier hashes (the "-" literals are dropped by + flatten); a plain name is a single hash. Rejoin on "-". (The hand + parser's identifier scanned hyphens in every identifier slot; the + corpus is hyphen-free outside run targets.)""" + if isinstance(tree, list): + return "-".join(_identifier(t) for t in tree) + return _identifier(tree) + + +def _text(v: object) -> str: + """Text of a captured literal keyword (primitive/function/case names...).""" + return v.content if isinstance(v, Slice) else v # type: ignore[return-value,union-attr] + + +class TreeToDocument: + def __init__(self, src: str, filename: str | None, resolve_dependency) -> None: + self._src = src + self._filename = filename + self._resolve_dependency = resolve_dependency + # from/to match spans (peg.py span recording), consumed in tree + # walk order — one (from, to) pair per compact AND block rule. + self._spans: list[tuple[str, int, int]] = [] + self._span_ix = 0 + + # -- entry -------------------------------------------------------------- + + def run(self, tree: Tree, spans: list[tuple[str, int, int]]) -> IscDocument: + self._spans = spans + system = _hash(_hash(tree)["system"]) + system_code = self.string(system["system_code"]) + + metadata: dict[str, Any] = {} + tests: list[IscTest] = [] + aliases: list[tuple[str, Any]] = [] + raw_stages: list[tuple[str, list[Any]]] = [] + raw_rulesets: dict[str, list[Any]] = {} + dependencies: list[IscDependency] = [] + languages: list[str] | None = None + + for entry in _array(system.get("body", [])): + h = _hash(entry) + if "metadata" in h: + metadata.update(self.metadata_fields(_array(h["metadata"]))) + continue + if "tests" in h: + tests = [self.test(t) for t in _array(h["tests"])] + continue + if "aliases" in h: + aliases = [ + (_identifier(ah["name"]), self.item(ah["value"])) + for ah in _array(h["aliases"]) + ] + continue + if "stage_name" in h and "stage" in h: + raw_stages.append( + (_identifier(h["stage_name"]), self.stage_body(_array(h["stage"]))) + ) + continue + if "ruleset_name" in h and "ruleset" in h: + name = _identifier(h["ruleset_name"]) + if name in raw_rulesets: + self._error(f"ruleset declared more than once: {name}") + raw_rulesets[name] = self.stage_body(_array(h["ruleset"])) + continue + if "target" in h: + alias = _identifier(h["alias"]) if "alias" in h else None + dependencies.append( + IscDependency(target=self.string(h["target"]), alias_name=alias) + ) + continue + if "languages" in h: + # Hand parity: `languages` declared more than once is an + # error; one declaration carries the name list (flatten's + # fold collapses a single language to a bare hash). + if languages is not None: + self._error("languages declared more than once") + lv = h["languages"] + languages = [ + _identifier(_hash(_hash(l)["language"])) + for l in (lv if isinstance(lv, list) else [lv]) + ] + continue + raise ValueError(f"unknown system body item {sorted(h.keys())}") + + # W1/W2 expansion (the same code path the hand parser used). + expander = _Expander( + system_code, + dependencies, + self._resolve_dependency, + self._src, + self._filename, + raw_rulesets, + ) + for name in raw_rulesets: + expander.expand_ruleset(name) + stages = [ + IscStage(name=name, items=tuple(expander.expand_items(items, (), False))) + for name, items in raw_stages + ] + + if self._span_ix != len(self._spans): + raise ValueError( + f"grammar/action mismatch: {len(self._spans) - self._span_ix} unconsumed " + f"from/to spans — the tree walk and span recording drifted" + ) + + return IscDocument( + system_code=system_code, + metadata=metadata, + tests=tuple(tests), + aliases=tuple(aliases), + stages=tuple(stages), + dependencies=tuple(dependencies), + languages=tuple(languages) if languages is not None else None, + rulesets={name: tuple(expander.expanded[name]) for name in raw_rulesets}, + ) + + def _error(self, msg: str) -> None: + from ..errors import IscParseError + + pos = len(self._src) + line = self._src.count("\n", 0, pos) + 1 + col = pos - self._src.rfind("\n", 0, pos) + full = f"{self._filename}: {msg}" if self._filename else msg + raise IscParseError(full, pos, line, col) + + # -- spans ---------------------------------------------------------------- + + def _next_span(self) -> tuple[int, int]: + """The (start, end) of the next recorded from/to capture.""" + if self._span_ix >= len(self._spans): + raise ValueError( + "grammar/action mismatch: no span recorded for a compact rule's " + "from/to — check peg.py span_names" + ) + _, start, end = self._spans[self._span_ix] + self._span_ix += 1 + return start, end + + # -- string literals -------------------------------------------------------- + + def string(self, tree: Tree) -> str: + """Decode a captured string-literal tree ({string: parts-or-slice}).""" + parts = _hash(tree)["string"] + if not isinstance(parts, list): + # single-quoted form: the whole body coalesced into one slice + return parts.content if isinstance(parts, Slice) else parts # type: ignore[return-value,union-attr] + out: list[str] = [] + for part in parts: + h = _hash(part) + if "run" in h: + out.append(_text(h["run"])) + elif "newline" in h: + out.append("\n") + elif "carriage_return" in h: + out.append("\r") + elif "tab" in h: + out.append("\t") + elif "dquote" in h: + out.append('"') + elif "backslash" in h: + out.append("\\") + elif "u_lone" in h: + out.append("U") + elif "unicode" in h: + digits = _text(h["unicode"]) # type: ignore[arg-type] + # DEVIATION (hand-parser parity): the grammar cannot + # distinguish `\uXXXX` from `\UXXXX` at the capture-name + # level (both are :unicode) — but the hand parser only + # DECODED lowercase \u and rendered a capital \U as a + # literal "U" followed by the raw digits. The slice offset + # tells us which spelling the source used (the escape + # letter sits one position before the digits). Mirroring + # the hand parser keeps documents byte-identical across + # the swap. (The ruby artifact parser decodes both + # spellings; aligning runtimes is an owner decision.) + if ord(self._src[h["unicode"].offset - 1]) == 0x55: + out.append("U" + digits) + else: + out.append(chr(int(digits, 16))) + else: + raise ValueError(f"unknown string part {sorted(h.keys())}") + return "".join(out) + + # -- items -------------------------------------------------------------------- + + def item(self, tree: Tree, to_position: bool = False) -> Any: + """A captured `item` — one atom, or a list of atoms (juxtaposition/+). + Mirrors the hand parser's parse_item: a single atom in `to` position + lowers a function keyword to a FuncallInline; several atoms form a + GroupItem.""" + v = _hash(tree)["concatenation"] + atoms: list[Any] = [] + kinds: list[str] = [] + for a in v if isinstance(v, list) else [v]: + atom, kind = self.atom(a) + atoms.append(atom) + kinds.append(kind) + if len(atoms) == 1: + return _function_to_inline(atoms[0], kinds[0]) if to_position else atoms[0] + return GroupItem(items=tuple(atoms)) + + def atom(self, tree: Tree) -> tuple[Any, str]: + """A single captured `item_atom` -> (IR item, parse kind). Function + atoms keep their kind so the `to` position can lower them to + FuncallInline (mirrors the hand parser's parse_item_atom_raw).""" + h = _hash(tree) + if "string" in h: + return StringItem(value=self.string(tree)), "string" + if "none" in h: + # converter parity: `none` lowers to the empty string + return StringItem(value=""), "string" + if "primitive" in h: + # DEVIATION (hand-parser parity): py renders primitives as + # AliasItems (the canonical schema maps them back by name). + return AliasItem(name=_text(h["primitive"])), "item" + if "any_char" in h: + return AliasItem(name="any_character"), "function" + if "function" in h: + return AliasItem(name=_text(h["function"])), "function" + if "any" in h: + return self.any(_hash(h["any"])), "item" + if "capture_inner" in h: + return CaptureGroupItem(data=self.item(h["capture_inner"])), "item" + if "maybe_inner" in h: + return RepeatItem(item=self.item(h["maybe_inner"]), min=0, max=1), "item" + if "some_inner" in h: + return RepeatItem(item=self.item(h["some_inner"]), min=1, max=None), "item" + if "ref" in h: + digit = _hash(h["ref"])["digit"] + # DEVIATION (grammar fidelity): the grammar pins ref(N) to a + # single digit; the hand parser's number scanner accepted + # multi-digit ids. The corpus is single-digit (the executor + # only addresses capture groups 1-9); dormant. + return CaptureRefItem(id=int(_text(digit))), "item" + if "alias" in h: + a = _hash(h["alias"]) + # DEVIATION (grammar fidelity, ts parity): a qualified alias + # reference inside items (`a.b`) is accepted by the grammar and + # spelled with the dot; the hand parser's identifier scan + # stopped at the dot and rejected the reference. The corpus is + # dotted-alias-free. + q = "." + _identifier(a["qualified_name"]) if "qualified_name" in a else "" + return AliasItem(name=_identifier(a) + q), "item" + raise ValueError(f"unknown item atom {sorted(h.keys())}") + + def any(self, h: Hash) -> Any: + if "single" in h: + # any("abc") — the hand parser splits a plain string into + # per-character string items. + v = self.string(h["single"]) + return AnyItem(of=tuple(StringItem(value=c) for c in v)) + if "lo" in h and "hi" in h: + return AnyCharClassItem(range=(self.string(h["lo"]), self.string(h["hi"]))) + if "list" in h: + # one list item collapses to a bare concatenation hash + lv = h["list"] + return AnyItem(of=tuple(self.item(t) for t in (lv if isinstance(lv, list) else [lv]))) + if "alias_ref" in h: + item, _ = self.atom({"alias": h["alias_ref"]}) + return AnyItem(of=(item,)) + if "any_item" in h: + # The DSL's `item` fallback inside any(...) — GRAMMAR_EXTENSIONS + # any_arg, hand-parser parity (`any(item)`, `any(item + item)`). + return AnyItem(of=(self.item(h["any_item"]),)) + raise ValueError(f"unknown any() shape {sorted(h.keys())}") + + # -- rules ---------------------------------------------------------------------- + + def rule(self, tree: Tree) -> ParsedRule: + h = _hash(tree) + constraints: list[tuple[str, Any]] = [] + for c in self.item_list(h.get("constraints")) or []: + entry = next(iter(_hash(c).items())) + constraints.append((entry[0], self.item(entry[1]))) + # The hand parser folds constraints into SubRule's four fixed + # fields, last kind-wins (written order between kinds is lost). + when_list = self.item_list(h.get("when")) + when = ( + tuple(_identifier(_hash(_hash(w)["when_lang"])) for w in when_list) + if when_list is not None + else None + ) + # Compact rules capture raw item_atoms (`sub "a" "b"`); block rules + # capture full items (`from "a"` — a concatenation tree). The + # concatenation key tells them apart. + if "concatenation" in _hash(h["from"]): + from_item = self.item(h["from"]) + self._next_span() # block rules carry no from_source + to_item = self.item(h["to"], to_position=True) + self._next_span() + from_source = None + else: + # Compact: from/to are single atoms; the W2 override key is the + # byte-exact source text of the from atom (the span peg.py + # recorded for this capture — identical to the hand parser's + # src[atom_start:atom_end].strip()). + from_item, _ = self.atom(h["from"]) + start, end = self._next_span() + to_item, to_kind = self.atom(h["to"]) + to_item = _function_to_inline(to_item, to_kind) + self._next_span() + from_source = self._src[start:end].strip() + return ParsedRule( + rule=_make_sub_rule(from_item, to_item, constraints, when), + from_source=from_source, + ) + + def item_list(self, v: Tree | None) -> list | None: + """A captured repetition's value: a list, or a single merged hash + when exactly one iteration produced a capture (flatten's sequence + fold drops the list wrapper). None when the capture is absent.""" + if v is None: + return None + return v if isinstance(v, list) else [v] + + def rule_list(self, trees: list) -> list: + """rule_items for parallel/sequence blocks: skip comment/noop rows; + `use` directives splice into the pass (GRAMMAR_EXTENSIONS + rule_items — the expansion resolves them in place).""" + out: list[Any] = [] + for t in trees: + h = _hash(t) + if "comment" in h or "noop" in h: + continue + if "use" in h: + out.append(self.use_item(_hash(h["use"]))) + else: + out.append(self.rule(t)) + return out + + # -- stage bodies ------------------------------------------------------------- + + def stage_body(self, trees: list) -> list[Any]: + out: list[Any] = [] + for tree in trees: + h = _hash(tree) + if "comment" in h or "noop" in h: + continue + if "bare_rule" in h: + out.append(self.rule(h["bare_rule"])) + continue + if "parallel" in h: + out.append(ParallelBlock(rules=tuple(self.rule_list(_array(h["parallel"]))))) + continue + if "sequence" in h: + out.append(SequenceBlock(rules=tuple(self.rule_list(_array(h["sequence"]))))) + continue + if "dep" in h and "stage" in h: + out.append( + RunItem(dependency=_dependency_name(h["dep"]), stage=_identifier(h["stage"])) + ) + continue + if "run_stage_only" in h: + out.append(RunItem(dependency=None, stage=_identifier(_hash(h["run_stage_only"])["stage"]))) + continue + if "separate" in h: + # Hand parity: a STRING separator is carried as a kwarg; a + # non-string separator atom is dropped entirely. + if "separator" in h: + atom_item, _ = self.atom(h["separator"]) + if isinstance(atom_item, StringItem): + out.append(FuncallRule(name="separate", kwargs={"separator": atom_item.value})) + continue + out.append(FuncallRule(name="separate")) + continue + if "case" in h: + # title_case's kwargs (case_kwargs) are captured by the + # grammar but ignored by the hand parser — mirrored here. + out.append(FuncallRule(name=_text(h["case"]))) + continue + if "compose" in h: + out.append(FuncallRule(name=_text(h["compose"]))) + continue + if "funcall_name" in h: + kwargs: dict[str, str] = {} + if "funcall_kwargs" in h: + # kwarg_list: one captured kwarg collapses to the bare + # hash; several arrive as a list of {kwarg: ...} hashes. + list_v = _hash(h["funcall_kwargs"])["kwarg"] + entries = ( + [_hash(e)["kwarg"] for e in list_v] + if isinstance(list_v, list) + else [_hash(list_v)] + ) + for kw in entries: + kwargs[_identifier(kw["kwarg_name"])] = self.string(kw["kwarg_value"]) + out.append(FuncallRule(name=_text(h["funcall_name"]), kwargs=kwargs)) + continue + if "use" in h: + out.append(self.use_item(_hash(h["use"]))) + continue + raise ValueError(f"unknown stage item {sorted(h.keys())}") + return out + + def use_item(self, u: Hash) -> RawUse: + """A captured use_directive: `use ` / `use .`, + optionally with an override block. Hand parser: `use a.b` names + the ruleset b, reached through dependency a.""" + name = _identifier(_hash(u["use_alias"])) + qualified = _identifier(u["qualified_name"]) if "qualified_name" in u else None + overrides = ( + tuple(self.rule_list(_array(u["use_overrides"]))) + if "use_overrides" in u + else None + ) + return RawUse( + name=qualified if qualified is not None else name, + dependency_alias=name if qualified is not None else None, + override=overrides, + ) + + # -- metadata ------------------------------------------------------------------- + + def metadata_fields(self, trees: list) -> dict[str, Any]: + metadata: dict[str, Any] = {} + for tree in trees: + h = _hash(tree) + if "field_name" in h: + key = _identifier(h["field_name"]) + if "field_value" in h: + metadata[key] = self.field_value(h["field_value"], h["field_name"]) + elif "field_block" in h: + metadata[key] = self.raw_text(h["field_block"]) + else: + metadata[key] = "" + continue + if "description" in h: + metadata["description"] = self.raw_text(h["description"]) + continue + if "notes" in h: + # note-list form; an empty block folds to [] — the hand + # parser's raw-text fallback yields "" (DEVIATION kept). + notes = [self.string(_hash(n)["note"]) for n in _array(h["notes"])] + metadata["notes"] = notes if notes else "" + continue + if "specification" in h or "provenance" in h: + # DEVIATION (grammar fidelity): the grammar decodes the + # comma-separated list; the hand parser only supported a + # single quoted value (multi-value hit its generic-field + # scanner error). The corpus is single-valued. + key = "specification" if "specification" in h else "provenance" + values = [self.string(_hash(v)[key]) for v in _array(h[key])] + metadata[key] = values[0] if len(values) == 1 else values + continue + if "relations" in h: + # DEVIATION (grammar fidelity): structured per the grammar; + # the hand parser rendered the whole block as raw text. + # The corpus declares no relations blocks (proven by the + # 434-file parity driver — a raw-vs-structured metadata + # value would diverge). + metadata["relations"] = [ + { + "type": _text(_hash(r)["type"]), + "system": self.string(_hash(r)["system"]), + **({"note": self.string(_hash(r)["note"])} if "note" in _hash(r) else {}), + } + for r in _array(h["relations"]) + ] + continue + if "system_status" in h: + metadata["system_status"] = _text(h["system_status"]) + continue + if "code_status" in h: + metadata["code_status"] = _text(h["code_status"]) + continue + # String-valued named fields: authority / name / identifying / + # source_spelling / target_spelling. Dispatched through + # field_value so a single-quoted value keeps the hand parser's + # raw-line rendering (see field_value's DEVIATION). + for key in ("authority", "name", "identifying", "source_spelling", "target_spelling"): + if key in h: + metadata[key] = self.field_value(h[key]) + break + return metadata + + def raw_text(self, v: object) -> str: + """The hand parser's parse_raw_text: a brace-block's raw text with + `\\{` and `\\}` unescaped, trimmed. An empty block folds to [] (a + named zero-iteration repetition) and reads as "".""" + if isinstance(v, Slice): + content = v.content + elif isinstance(v, str): + content = v + elif isinstance(v, list) and not v: + return "" + else: + raise ValueError(f"unknown raw text {v!r}") + return content.replace("\\{", "{").replace("\\}", "}").strip() + + def field_value(self, tree: Tree, field_name: Tree | None = None) -> str: + h = _hash(tree) + if "raw" in h: + return _text(h["raw"]).strip() # type: ignore[arg-type] + # DEVIATION (hand-parser parity): the hand parser only treated a + # DOUBLE-quoted value as a quoted string — anything else + # (single-quoted values, bare words) fell through to its + # read-to-newline fallback, keeping quotes/spelling verbatim, + # trimmed. The grammar decodes single-quoted values properly, so + # dispatch on the SOURCE spelling at the position the hand parser + # would read from (just past the field name and its inline + # whitespace): `"` decodes, everything else takes the raw line. + start = self.value_start(h, field_name) + if start is not None and self._src[start : start + 1] != '"': + return self.raw_line_from(start).strip() + return self.string(tree) + + def value_start(self, h: Hash, field_name: Tree | None) -> int | None: + """The position the hand parser's value reader starts from: right + after the field name plus inline whitespace. Anchored on the value + slice when there is one (the opening quote sits just before it), + otherwise on the field name.""" + parts = h.get("string") + if isinstance(parts, list): + if parts and parts[0] is not None and isinstance(parts[0], dict): + run = parts[0].get("run") + if isinstance(run, Slice): + return run.offset - 1 + if not parts and field_name is not None: + identifier = _hash(field_name)["identifier"] + if isinstance(identifier, Slice): + p = identifier.offset + len(identifier.content) + while p < len(self._src) and self._src[p] in " \t": + p += 1 + return p + return None + if isinstance(parts, Slice): + return parts.offset - 1 + return None + + def raw_line_from(self, pos: int) -> str: + """The hand parser's parse_to_newline: from `pos` to the next newline.""" + nl = self._src.find("\n", pos) + return self._src[pos : nl if nl != -1 else None] + + def test(self, tree: Tree) -> IscTest: + th = _hash(tree) + note = self.string(th["note"]) if "note" in th else None + return IscTest(input=self.string(th["input"]), expected=self.string(th["expected"]), note=note) diff --git a/interscript-py/src/interscript/grammar/grammar_source.py b/interscript-py/src/interscript/grammar/grammar_source.py new file mode 100644 index 00000000..a0ef33aa --- /dev/null +++ b/interscript-py/src/interscript/grammar/grammar_source.py @@ -0,0 +1,27 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Vendored copy of the canonical ISC grammar: + + interscript-ruby lib/interscript/isc/grammar/isc.parg + +The .parg file is the single semantic source of the ISC syntax; this +module embeds it verbatim (sha256 b0f7a7caf462a1671a80ef974a59425c636f1d32cb3e405280991b21a6edd712) so the Python parser compiles +the same grammar text the Ruby runtime ships. Update procedure: + + python scripts/sync_isc_grammar.py + +The sync script rewrites this file and the sha256 pin; the pin test +(tests/test_grammar_source.py) fails when the embedded text and the +recorded hash disagree, so drift is never silent. + +This is the same file (same sha256 pin) that interscript-ts vendors via +its scripts/sync-isc-grammar.mjs — the two generated parsers must never +compile different grammar texts. +""" + +from __future__ import annotations + +# sha256 of the grammar text embedded below (pin for drift detection). +ISC_PARG_SHA256 = "b0f7a7caf462a1671a80ef974a59425c636f1d32cb3e405280991b21a6edd712" + +# The grammar text itself, byte-identical to the ruby checkout copy. +ISC_PARG_SOURCE = "# Interscript ISC grammar \u2014 parsanol grammar language (PG).\n#\n# Line-oriented port of lib/interscript/isc/grammar/concerns/*.rb.\n# The capture tree must reproduce the Ruby-DSL parser's tree exactly:\n# - repetitions are ABNF prefixes (1*x, *x)\n# - negation is !x, positive lookahead is &x\n# - captures at use sites (rule_ref as name) nest inner captures\n# the same way parslet .as does\n#\n# Consumed by the interscript gem (artifact parse) and parsanol npm.\n\ngrammar InterscriptIsc version \"0.1.0\" {\n\n # -- lexical primitives -------------------------------------------------\n\n chr = %x00-10FFFF\n space_char = %x20 / %x09 / %x0A / %x0B / %x0C / %x0D\n newline = %x0D.0A / %x0A / %x0D\n line_comment = \"#\" *(!(newline) chr)\n whitespace = 1*(space_char / line_comment)\n ws = *(space_char / line_comment)\n inline_space = 1*(%x20 / %x09)\n inline_ws = *(inline_space / line_comment)\n hexd = %x30-39 / %x41-46 / %x61-66\n digit = %x30-39\n\n ident_first = %x41-5A / %x61-7A / %x5F\n ident_rest = %x41-5A / %x61-7A / %x30-39 / %x5F\n identifier = (ident_first 0*ident_rest) as identifier\n\n keyword = \"parallel\" / \"sequence\" / \"stage\" / \"compose\" / \"separate\" / \"system\" / \"metadata\" / \"aliases\" / \"tests\" / \"notes\" / \"description\" / \"any_character\" / \"authority\" / \"dependency\" / \"run\" / \"sub\" / \"before\" / \"after\" / \"not_before\" / \"not_after\" / \"any\" / \"none\" / \"boundary\" / \"line_start\" / \"line_end\" / \"word_boundary\" / \"downcase\" / \"upcase\" / \"title_case\" / \"capture\" / \"maybe\" / \"some\" / \"ref\" / \"name\" / \"ruleset\" / \"use\" / \"override\" / \"languages\" / \"when\"\n\n # -- string literals ----------------------------------------------------\n\n esc_nl = (\"n\") as newline\n esc_cr = (\"r\") as carriage_return\n esc_tab = (\"t\") as tab\n esc_dq = \"\\\"\" as dquote\n esc_bs = \"\\\\\" as backslash\n esc_uni4 = \"u\" (4hexd) as unicode\n esc_uni8 = \"U\" (4*8hexd) as unicode\n esc_u_lone = \"U\" as u_lone\n escape_sequence = \"\\\\\" (esc_nl / esc_cr / esc_tab / esc_dq / esc_bs / esc_uni4 / esc_uni8 / esc_u_lone)\n\n raw_run = 1*(!(\"\\\\\" / \"\\\"\") chr)\n dq_parts = *(escape_sequence / (raw_run as run))\n double_quoted = \"\\\"\" (dq_parts as string) \"\\\"\"\n sq_body = *(!\"'\" chr)\n single_quoted = \"'\" (sq_body as string) \"'\"\n quoted_string = double_quoted / single_quoted\n\n arrow = [whitespace] \"->\" [whitespace]\n comma_ws = \",\" [whitespace]\n\n # -- metadata -----------------------------------------------------------\n\n md_authority = \"authority\" whitespace ((quoted_string) as authority)\n md_source_spelling = \"source_spelling\" whitespace ((quoted_string) as source_spelling)\n md_target_spelling = \"target_spelling\" whitespace ((quoted_string) as target_spelling)\n md_identifying = \"identifying\" whitespace ((quoted_string) as identifying)\n md_name = \"name\" whitespace ((quoted_string) as name)\n md_spec_value = quoted_string as specification\n md_specification = \"specification\" whitespace ((md_spec_value *(comma_ws md_spec_value)) as specification)\n md_description = \"description\" [whitespace] \"{\" [ws] (raw_text as description) [ws] \"}\"\n md_system_status = \"system_status\" whitespace ((\"current\" / \"former\" / \"inactive\") as system_status)\n md_code_status = \"code_status\" whitespace ((\"preferred\" / \"proposed\" / \"deprecated\") as code_status)\n\n relation_type = \"supersedes\" / \"superseded_by\" / \"based_on\" / \"basis_for\" / \"alias_of\" / \"adopted_from\" / \"related_to\"\n relation_note = whitespace \"note\" whitespace (quoted_string as note)\n relation_item = [whitespace] (relation_type as type) whitespace (quoted_string as system) [(relation_note)] [whitespace]\n md_relations = \"relations\" [whitespace] \"{\" [ws] ((*(relation_item)) as relations) [ws] \"}\"\n\n note_item = [whitespace] \"note\" [whitespace] (quoted_string as note) [whitespace]\n md_notes = \"notes\" [whitespace] \"{\" [ws] ((*(note_item)) as notes) [ws] \"}\"\n\n md_provenance_value = quoted_string as provenance\n md_provenance = \"provenance\" [whitespace] ((md_provenance_value *(comma_ws md_provenance_value)) as provenance)\n\n md_empty_field = &(newline)\n md_raw_field_value = 1*(!%x0A !\"}\" !\"{\" chr)\n md_field_value = quoted_string / (md_raw_field_value as raw)\n md_field_block = \"{\" [ws] (raw_text as field_block) [ws] \"}\"\n md_generic_field = (identifier as field_name) inline_ws ((md_field_value as field_value) / md_field_block / md_empty_field)\n\n metadata_field = [whitespace] (md_description / md_relations / md_system_status / md_code_status / md_specification / md_notes / md_provenance / md_generic_field) [whitespace]\n md_items = *(metadata_field)\n metadata_block = \"metadata\" [whitespace] \"{\" [ws] (md_items as metadata) [ws] \"}\"\n\n raw_text = *(\"\\\\\" chr / (!\"}\" chr))\n\n # -- aliases ------------------------------------------------------------\n\n alias_decl = [whitespace] (identifier as name) [whitespace] \"=\" [whitespace] (item as value) [whitespace]\n alias_items = *(alias_decl)\n aliases_block = \"aliases\" [whitespace] \"{\" [ws] (alias_items as aliases) [ws] \"}\"\n\n # -- dependencies -------------------------------------------------------\n\n dep_alias = whitespace \"as\" whitespace (identifier as alias)\n dependency_decl = \"dependency\" whitespace (quoted_string as target) [(dep_alias)]\n\n # -- item expressions ---------------------------------------------------\n\n zw_primitive = ((\"boundary\") / (\"line_start\") / (\"line_end\") / (\"word_boundary\") / (\"space\") / (\"non_boundary\")) as primitive\n fn_lit = ((\"upcase\") / (\"downcase\") / (\"title_case\") / (\"reverse\") / (\"strip\") / (\"swapcase\")) as function\n any_char_lit = (\"any_character\") as any_char\n none_lit = (\"none\") as none\n\n digit_cap = digit as digit\n capture_reference_body = \"ref\" \"(\" [ws] digit_cap [ws] \")\"\n capture_reference = capture_reference_body as ref\n\n item_ref = identifier \".\" (identifier as qualified_name)\n alias_reference = (!(keyword) identifier [(\".\" (identifier as qualified_name))] !\"{\" ) as alias\n\n item_seq = item_atom\n item_sep = ([ws] \"+\" [ws]) / whitespace\n item_cont = &item_atom\n\n range_arg = (quoted_string as lo) [ws] \"..\" [ws] (quoted_string as hi)\n list_item = item\n list_items = (list_item *((comma_ws / whitespace) list_item))\n set_arg = ((quoted_string as single)) / (\"[\" [ws] ((list_item *((comma_ws / whitespace) list_item)) as list) [ws] \"]\")\n alias_arg = (!zw_primitive !(keyword) identifier) as alias_ref\n any_arg = range_arg / set_arg / alias_arg\n any_ctor = \"any\" \"(\" [ws] (any_arg as any) [ws] \")\"\n capture_ctor = \"capture\" \"(\" [ws] (item as capture_inner) [ws] \")\"\n maybe_ctor = \"maybe\" \"(\" [ws] (item as maybe_inner) [ws] \")\"\n some_ctor = \"some\" \"(\" [ws] (item as some_inner) [ws] \")\"\n\n item_atom_start = \"\\\"\" / \"'\" / ident_first\n cont_stop = \"to\" / \"before\" / \"after\" / \"not_before\" / \"not_after\" / \"}\"\n alias_decl_stop = identifier [ws] \"=\"\n item_atom = quoted_string / none_lit / zw_primitive / any_char_lit / any_ctor / capture_ctor / maybe_ctor / some_ctor / fn_lit / capture_reference / alias_reference\n item = (item_atom 0*(item_sep !cont_stop !(alias_decl_stop) &item_atom_start item_atom)) as concatenation\n\n # -- rules and stages ---------------------------------------------------\n\n constraint_after = \"after\" whitespace (item as after)\n constraint_before = \"before\" whitespace (item as before)\n constraint_not_before = \"not_before\" whitespace (item as not_before)\n constraint_not_after = \"not_after\" whitespace (item as not_after)\n constraint = constraint_after / constraint_before / constraint_not_before / constraint_not_after\n constraints = *(whitespace constraint)\n\n compact_rule = \"sub\" whitespace (item_atom as from) whitespace (item_atom as to) ((*(whitespace constraint)) as constraints) [(when_guard)]\n\n block_rule = \"sub\" [ws] \"{\" [ws] \"from\" whitespace (item as from) [ws] \"to\" whitespace (item as to) ((*(whitespace constraint)) as constraints) [(when_guard)] [ws] \"}\"\n\n when_lang = (!(keyword) identifier) as when_lang\n when_guard = (whitespace \"when\" whitespace (when_lang) *(((comma_ws / whitespace) (when_lang)))) as when\n\n rule = block_rule / compact_rule\n\n comment_item = ((\"#\" *(!%x0A chr)) as comment) / ((!(keyword) identifier) as noop)\n rule_line = [ws] (rule / comment_item) [ws]\n rule_items = *(rule_line)\n sequence_block = \"sequence\" [ws] \"{\" [ws] (rule_items as sequence) [ws] \"}\"\n parallel_block = \"parallel\" [ws] \"{\" [ws] (rule_items as parallel) [ws] \"}\"\n\n separator_part = whitespace \"separator\" whitespace (item_atom as separator)\n separate_directive = (\"separate\") as separate [(separator_part)]\n case_directive = ((\"downcase\") / (\"upcase\")) as case\n case_kwargs_part = whitespace (kwarg_list as case_kwargs)\n title_case_directive = (\"title_case\") as case [(case_kwargs_part)]\n string_case_directive = case_directive / title_case_directive\n compose_directive = ((\"compose\") / (\"decompose\")) as compose\n kwarg = (identifier as kwarg_name) [ws] \":\" [ws] ((quoted_string) as kwarg_value)\n kwarg_item = (kwarg) as kwarg\n kwarg_list = (kwarg_item *(comma_ws kwarg_item))\n funcall_kwargs_part = whitespace (kwarg_list as funcall_kwargs)\n rababa_directive = (\"rababa\") as funcall_name [(funcall_kwargs_part)]\n run_dep = \"map.\" (identifier as dep) \".stage.\" (identifier as stage)\n run_stage_only = (\"stage.\" (identifier as stage)) as run_stage_only\n run_rule = \"run\" whitespace ((run_dep) / run_stage_only)\n\n bare_rule = (rule) as bare_rule\n\n use_alias = (identifier) as use_alias\n use_qualified = \".\" (identifier as qualified_name)\n use_overrides_part = whitespace \"override\" [ws] \"{\" [ws] ((*(rule_line)) as use_overrides) [ws] \"}\"\n use_directive = ((\"use\" whitespace (use_alias) [(use_qualified)] [(use_overrides_part)]) as use)\n\n stage_item = [ws] (sequence_block / parallel_block / run_rule / separate_directive / string_case_directive / compose_directive / rababa_directive / use_directive / bare_rule / comment_item) [ws]\n stage_items = *(stage_item)\n stage_name_paren = \"(\" (identifier as stage_name) \")\"\n stage_name_ws = whitespace (identifier as stage_name)\n stage_block = \"stage\" (stage_name_paren / stage_name_ws) [ws] \"{\" [ws] (stage_items as stage) [ws] \"}\"\n\n ruleset_name_ws = whitespace (identifier as ruleset_name)\n ruleset_block = \"ruleset\" (ruleset_name_ws) [ws] \"{\" [ws] (stage_items as ruleset) [ws] \"}\"\n\n # -- tests ---------------------------------------------------------------\n\n test_note = whitespace \"note\" whitespace (quoted_string as note)\n test_line = [whitespace] (quoted_string as input) arrow (quoted_string as expected) [(test_note)] [whitespace]\n test_items = *(test_line)\n tests_block = \"tests\" [whitespace] \"{\" [ws] (test_items as tests) [ws] \"}\"\n\n # -- system --------------------------------------------------------------\n\n language_ident = (!(keyword) identifier) as language\n languages_decl = (\"languages\" whitespace ((language_ident *(((comma_ws / whitespace) (language_ident))))) as languages)\n\n block_item = [whitespace] (metadata_block / aliases_block / tests_block / languages_decl / ruleset_block / stage_block / dependency_decl) [whitespace]\n body_items = *(block_item)\n system_block = \"system\" whitespace ((quoted_string) as system_code) [whitespace] \"{\" [ws] ((*(block_item)) as body) [ws] \"}\" [whitespace]\n isc_source = [whitespace] ((system_block) as system) [whitespace]\n\n entry isc: isc_source\n}\n" diff --git a/interscript-py/src/interscript/grammar/isc_parser.py b/interscript-py/src/interscript/grammar/isc_parser.py new file mode 100644 index 00000000..53030ea0 --- /dev/null +++ b/interscript-py/src/interscript/grammar/isc_parser.py @@ -0,0 +1,180 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Generated ISC parser — the parsanol grammar (grammar_source.py) drives a +table-driven PEG interpreter (parg_reader.py + peg.py) and the resulting +capture tree is lowered to an IscDocument by the action layer +(actions.py), which also runs the W1/W2 expansion (isc._Expander). + +This module replaces the hand-written recursive-descent parser while +keeping its public API byte-compatible: `parse_isc(source, filename, +resolve_dependency)` returning an IscDocument and throwing IscParseError +on syntax errors. + +Single-sourcing contract: the grammar TEXT embedded in grammar_source.py +is the semantic source of truth (synced from interscript-ruby's +lib/interscript/isc/grammar/isc.parg via scripts/sync_isc_grammar.py). +The four documented grammar-table extensions below (GRAMMAR_EXTENSIONS) +restore hand-parser behaviors the canonical .parg rejects — each is +spelled in the grammar language, named, and corpus-proven dormant. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from ..errors import IscParseError +from .actions import TreeToDocument +from .grammar_source import ISC_PARG_SOURCE +from .parg_reader import Alt, Lit, PargGrammar, parse_parg +from .peg import PegError, PegInterpreter + +# The compact rule's from/to captures carry no delimiters in the capture +# tree (closing quotes are not captured) — but the W2 override key is the +# byte-exact source text of the from atom. The interpreter records the +# exact match spans for these two capture names; the action layer pairs +# them with the tree in walk order. +_SPAN_CAPTURES = frozenset({"from", "to"}) + +# extract [A-Za-z0-9_]* at a position (the hand parser's peek_word) +_WORD_RE = re.compile(r"[A-Za-z0-9_]*") + + +@dataclass +class CompiledGrammar: + grammar: PargGrammar + # rule name -> peg node (the merged table: canonical .parg + extensions) + rules: dict + entry: str + # The keyword table, derived from the grammar itself (load-bearing for + # the keyword-in-item-position diagnostic below — no drift). + keywords: frozenset[str] = field(default_factory=frozenset) + + +_compiled: CompiledGrammar | None = None + + +def compile_grammar() -> CompiledGrammar: + global _compiled + if _compiled is not None: + return _compiled + + grammar = parse_parg(ISC_PARG_SOURCE) + rules = dict(grammar.rules) + # Shape-guarded merge: if a future .parg edit renames any of the three + # rules, the merge raises instead of silently misparsing. + ext = parse_parg(GRAMMAR_EXTENSIONS) + for name in ("any_arg", "block_rule", "run_dep", "rule_items"): + if name not in rules or name not in ext.rules: + raise RuntimeError( + f"grammar extension target {name} missing from isc.parg — " + f"recheck GRAMMAR_EXTENSIONS" + ) + rules[name] = ext.rules[name] + kw_rule = grammar.rules["keyword"] + keywords = ( + frozenset(n.s for n in kw_rule.a if isinstance(n, Lit)) + if isinstance(kw_rule, Alt) + else frozenset() + ) + _compiled = CompiledGrammar( + grammar=grammar, + rules=rules, + entry=grammar.entries.get("isc", "isc_source"), + keywords=keywords, + ) + return _compiled + + +def parse_isc(source, filename=None, resolve_dependency=None): + """Parse ISC source text into an IscDocument. + + doc = parse_isc(source, "map.isc") + + Raises IscParseError (with filename/line/col, like the hand parser) + on syntax errors. + """ + c = compile_grammar() + interp = PegInterpreter(c.rules, source, span_names=_SPAN_CAPTURES) + try: + tree, _end = interp.parse_root(c.entry) + except PegError as e: + message, at = _hand_style_failure(source, c, interp, e) + at = max(min(at, max(len(source) - 1, 0)), 0) + line = source.count("\n", 0, at) + 1 + col = at - source.rfind("\n", 0, at) + full = f"{filename}: {message}" if filename else message + raise IscParseError(full, at, line, col) from None + return TreeToDocument(source, filename, resolve_dependency).run(tree, interp.spans) + + +def _hand_style_failure(source, c, interp, peg_error) -> tuple[str, int]: + """Render a parse failure the way the replaced hand parser reported + it. Two hand-parser messages are pinned by the suite and reproduced + from real parse state; everything else gets the interpreter's + deepest-failure report (the same contract as the hand parser's + deepest-error reporting). Returns (message, position).""" + if peg_error.at >= len(source) and "double_quoted" in interp.failed_rules: + # A double-quoted literal ran past EOF (the hand parser's + # parse_quoted_string reports this exact message). + return "Unterminated string", peg_error.at + # A keyword landed in item position: item_atom failed at a position + # whose word is a grammar keyword — the hand parser's + # parse_item_atom_raw guard, verbatim message and position. (The + # interpreter's farthest-failure position cannot be used here: the + # alias_decl_stop lookahead scans past the keyword first.) + for pos in interp.failed_positions.get("item_atom", []): + m = _WORD_RE.match(source, pos) + word = m.group(0) if m else "" + if word in c.keywords: + return f"Unexpected keyword in item position: {word}", pos + return peg_error.args[0], peg_error.at + + +GRAMMAR_EXTENSIONS = """ +grammar IscExtensions version "0.0.0" { + any_arg = range_arg / set_arg / alias_arg / ((item) as any_item) + block_rule = ("sub" [ws] "{" [ws] "from" whitespace ((item) as from) [ws] "to" whitespace ((item) as to) ((*(whitespace constraint)) as constraints) [ws] "}" [(when_guard)]) / ("sub" [ws] "{" [ws] "from" whitespace ((item) as from) [ws] "to" whitespace ((item) as to) ((*(whitespace constraint)) as constraints) [(when_guard)] [ws] "}") + run_dep = "map." (((identifier *("-" identifier)) / identifier) as dep) ".stage." (identifier as stage) + rule_items = *(([ws] ((use_directive / rule / comment_item)) [ws])) +} +""" + +# GRAMMAR_EXTENSIONS — the places where the former hand parser accepted +# syntax the canonical .parg rejects. Each is spelled in the grammar +# language itself and merged into the rule table, so the .parg remains +# the only semantic source and every extension is one visible, named +# block (the first three are the same extensions the ts generated +# parser ships): +# +# 1. `any_arg` — restore the ruby DSL's final `item as :any_item` +# fallback (concerns/items.rb any_constructor), which the canonical +# .parg dropped; the interscript-rs vendored copy keeps it. +# (`any(space)`, `any(space + line_end)` — the ts suite pins both.) +# 2. `block_rule` — accept the `when` guard AFTER the closing brace of +# the block form. The ruby DSL places the guard inside the braces; +# the ts and py runtimes both place it outside, and the py suite pins +# the outside form (test_isc_parser.py::test_when_suffix_after_block_form). +# The outside branch is tried first (it is the greedy one; its +# mandatory `}` keeps it from shadowing the inside form). +# 3. `run_dep` — allow hyphenated dependency names (`run map.some-dep.…`). +# The grammar's identifier is [A-Za-z_][A-Za-z0-9_]*; the former hand +# parser's identifier admitted `-` in structural positions. Only the +# dependency slot is widened; the corpus is hyphen-free in every +# identifier position (proven by the corpus parity driver). +# 4. `rule_items` — accept `use` directives inside parallel/sequence +# blocks (ISC-WF-1 §3: a ruleset can be spliced into one pass). The +# canonical .parg only feeds rule_line (sub-rows/comments) to pass +# blocks; the hand parser accepted `use` there and the py suite pins +# it (test_isc_parser.py::test_use_inside_parallel_participates_in_pass). +# `use` shadows nothing in rule position (rule always starts with +# "sub", comment with "#"). The override block's rule_line is left +# canonical — the hand parser silently DROPPED non-sub rows there, +# so a `use` inside `override {}` is a generated-parser rejection of +# a construct the corpus never uses. +# +# Ruby rejects (1)'s absence aside, the outside-when, hyphen-dep, and +# use-in-pass forms are cross-runtime divergences (ruby parses none); +# aligning ruby/ts/py on one placement is an owner decision. None of the +# four extensions fires on any corpus map (proven: the corpus py-vs-py +# document-diff driver passes byte-identical with them in place, and a +# corpus-wide scan finds no `use` inside a pass block). diff --git a/interscript-py/src/interscript/grammar/parg_reader.py b/interscript-py/src/interscript/grammar/parg_reader.py new file mode 100644 index 00000000..4837d172 --- /dev/null +++ b/interscript-py/src/interscript/grammar/parg_reader.py @@ -0,0 +1,426 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Reader for the parsanol grammar language (PARG) — the text format the +canonical ISC grammar (isc.parg) is written in. + +This is a Python port of the relevant subset of Parsanol::PARG +(lexer.rb + parser.rb + compiler.rb's parse_hex), mirrored from +interscript-ts's src/isc/grammar/parg-reader.ts and pinned to the +constructs the ISC grammar uses: + + - the `grammar version "..." { ... }` envelope + - `entry : ` declarations + - rules: `name = expression`, terminated by newline (ABNF/PEG mix: + ordered choice `/`, sequences, `[x]` optional, `n*m` repetition, + `!x`/`&x` lookaheads, `expr as name` captures, string literals, + `%xNN-NN` character ranges, rule references) + +Anything outside that subset (states, tables, preprocess, render, +derive, brace rule bodies, `%i`/`%s` literals, categories) raises a +PargParseError naming the construct — the reader fails loudly instead +of silently misparsing a grammar it does not understand. + +The AST node kinds mirror the compiled atom graph (see peg.py for the +interpreter and the flattening semantics it reproduces): + + Lit(s) literal string + CharClass(ranges) single character class + Seq(parts) ordered sequence + Alt(branches) ordered choice (PEG) + Rep(body, min, max) repetition (max None = open) + Pred(positive, body) lookahead, zero-width + Cap(name, body) named capture + Ref(name) rule reference +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +class PargParseError(ValueError): + """The PARG text itself is malformed (or uses an unsupported construct).""" + + +# --------------------------------------------------------------------------- +# AST +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class Lit: + s: str + + +@dataclass(frozen=True, slots=True) +class CharClass: + ranges: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True, slots=True) +class Seq: + a: tuple + + +@dataclass(frozen=True, slots=True) +class Alt: + a: tuple + + +@dataclass(frozen=True, slots=True) +class Rep: + a: object + min: int + max: int | None + maybe: bool = False + + +@dataclass(frozen=True, slots=True) +class Pred: + positive: bool + a: object + + +@dataclass(frozen=True, slots=True) +class Cap: + name: str + a: object + + +@dataclass(frozen=True, slots=True) +class Ref: + name: str + + +@dataclass +class PargGrammar: + name: str = "" + version: str = "" + # entry name -> root rule name (the ISC grammar declares `entry isc: isc_source`) + entries: dict[str, str] = field(default_factory=dict) + # rule name -> AST + rules: dict[str, object] = field(default_factory=dict) + # declaration order, for diagnostics + order: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Lexer — token classes mirror Parsanol::PARG::Lexer (comments and +# whitespace dropped, newlines kept as separators). +# --------------------------------------------------------------------------- + +_HEX_RE = re.compile(r"%x[0-9A-Fa-f]{2,6}(?:-[0-9A-Fa-f]{2,6}|(?:\.[0-9A-Fa-f]{2,6})+)?") +_STR_RE = re.compile(r'"(?:[^"\\]|\\.)*"') +_SSTR_RE = re.compile(r'%s"(?:[^"\\]|\\.)*"') +_NUM_RE = re.compile(r"\d+") +_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_PUNCT_RE = re.compile(r"[=/()\[\]{}*!&:,.<-]") + +# (type, value, offset) — offset into the source text, for diagnostics + + +def _lex(text: str) -> list[tuple[str, str, int]]: + tokens: list[tuple[str, str, int]] = [] + pos = 0 + n = len(text) + while pos < n: + c = text[pos] + if c == "\n" or (c == "\r" and text[pos + 1 : pos + 2] == "\n"): + tokens.append(("newline", "\n", pos)) + pos += 2 if c == "\r" else 1 + continue + if c in " \t": + pos += 1 + continue + if c == "#": + while pos < n and text[pos] != "\n": + pos += 1 + continue + if c == "%": + if text.startswith('%s"', pos): + m = _SSTR_RE.match(text, pos) + if m is None: + raise PargParseError(f'unterminated %s"..." literal at offset {pos}') + tokens.append(("sstr", m.group(0)[3:-1], pos)) + pos = m.end() + continue + if text.startswith('%i"', pos): + raise PargParseError( + f'%i"..." case-insensitive literals are not supported by this reader (offset {pos})' + ) + if text.startswith("%p{", pos): + raise PargParseError( + f"%p{{...}} category classes are not supported by this reader (offset {pos})" + ) + m = _HEX_RE.match(text, pos) + if m is not None: + tokens.append(("hex", m.group(0)[2:], pos)) + pos = m.end() + continue + raise PargParseError(f"unexpected %-construct at offset {pos}") + if c == '"': + m = _STR_RE.match(text, pos) + if m is None: + raise PargParseError(f"unterminated string literal at offset {pos}") + tokens.append(("str", m.group(0)[1:-1], pos)) + pos = m.end() + continue + if c in "0123456789": + # ASCII digits only (Python's str.isdigit is Unicode-aware; the + # PARG number token is %x30-39, mirroring the ts lexer's \d). + m = _NUM_RE.match(text, pos) + tokens.append(("num", m.group(0), pos)) + pos = m.end() + continue + if ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "_": + m = _IDENT_RE.match(text, pos) + tokens.append(("ident", m.group(0), pos)) + pos = m.end() + continue + m = _PUNCT_RE.match(text, pos) + if m is not None: + tokens.append(("punct", m.group(0), pos)) + pos += 1 + continue + raise PargParseError(f"unexpected character {c!r} at offset {pos}") + return tokens + + +def _unquote(raw: str) -> str: + """Unquote a literal body: \\n \\t \\r are escapes, anything else is itself.""" + out: list[str] = [] + i = 0 + n = len(raw) + while i < n: + c = raw[i] + if c != "\\": + out.append(c) + i += 1 + continue + i += 1 + nxt = raw[i] if i < n else "" + i += 1 + if nxt == "n": + out.append("\n") + elif nxt == "t": + out.append("\t") + elif nxt == "r": + out.append("\r") + else: + out.append(nxt) + return "".join(out) + + +# --------------------------------------------------------------------------- +# Parser — mirrors Parsanol::PARG::Parser productions for the supported +# subset. Sequences end at a newline (outside `(...)` grouping, which the +# ISC grammar never spans), `/`, `)`, `]` or `}`. +# --------------------------------------------------------------------------- + + +class _PargParser: + def __init__(self, text: str) -> None: + self._tokens = _lex(text) + self._pos = 0 + self.grammar = PargGrammar() + + def _peek(self) -> tuple[str, str, int] | None: + return self._tokens[self._pos] if self._pos < len(self._tokens) else None + + def _advance(self) -> tuple[str, str, int]: + if self._pos >= len(self._tokens): + raise PargParseError("unexpected end of grammar") + tok = self._tokens[self._pos] + self._pos += 1 + return tok + + def _expect(self, type_: str, value: str | None = None) -> tuple[str, str, int]: + tok = self._advance() + if tok[0] != type_ or (value is not None and tok[1] != value): + found = f"{tok[0]}:{tok[1]!r}" + expected = repr(value) if value is not None else type_ + raise PargParseError(f"expected {expected}, got {found} at offset {tok[2]}") + return tok + + def _skip_newlines(self) -> None: + while (tok := self._peek()) is not None and tok[0] == "newline": + self._pos += 1 + + def parse(self) -> PargGrammar: + self._skip_newlines() + while self._pos < len(self._tokens): + section = self._advance() + if section[0] != "ident": + raise PargParseError( + f"expected a section (grammar/entry), got {section[0]} at offset {section[2]}" + ) + if section[1] == "grammar": + self._parse_grammar() + elif section[1] == "entry": + self._parse_entry() + else: + raise PargParseError( + f'unsupported PARG section "{section[1]}" at offset {section[2]} ' + f"(this reader implements grammar/entry only)" + ) + self._skip_newlines() + return self.grammar + + def _parse_grammar(self) -> None: + self.grammar.name = self._expect("ident")[1] + self._expect("ident", "version") + self.grammar.version = _unquote(self._expect("str")[1]) + # `edition` is accepted and ignored when present (RS vendored copy + # carries `edition "v1"`; the canonical ISC grammar omits it). + tok = self._peek() + if tok is not None and tok[0] == "ident" and tok[1] == "edition": + self._advance() + tok = self._advance() + if tok[0] not in ("str", "num"): + raise PargParseError(f'expected edition value after "edition" at offset {tok[2]}') + self._expect("punct", "{") + self._skip_newlines() + while not ((tok := self._peek()) is not None and tok[0] == "punct" and tok[1] == "}"): + if self._peek() is None: + raise PargParseError("unterminated grammar block") + if (tok := self._peek())[0] == "ident" and tok[1] == "entry": + self._advance() + self._parse_entry() + self._skip_newlines() + continue + self._parse_rule() + self._skip_newlines() + self._expect("punct", "}") + + def _parse_entry(self) -> None: + name = self._expect("ident")[1] + self._expect("punct", ":") + tok = self._expect("ident") + self.grammar.entries[name] = tok[1] + + def _parse_rule(self) -> None: + name = self._expect("ident")[1] + self._expect("punct", "=") + if name in self.grammar.rules: + raise PargParseError(f"duplicate rule {name!r}") + self.grammar.rules[name] = self._parse_choice() + self.grammar.order.append(name) + + def _stop_element(self) -> bool: + tok = self._peek() + if tok is None: + return True + if tok[0] == "newline": + return True + return tok[0] == "punct" and tok[1] in ("/", ")", "]", "}") + + def _parse_choice(self) -> object: + branches = [self._parse_seq()] + while (tok := self._peek()) is not None and tok[0] == "punct" and tok[1] == "/": + self._advance() + branches.append(self._parse_seq()) + return branches[0] if len(branches) == 1 else Alt(tuple(branches)) + + def _parse_seq(self) -> object: + if self._stop_element(): + tok = self._peek() + raise PargParseError(f"empty sequence at offset {tok[2] if tok else -1}") + items = [self._parse_element()] + while not self._stop_element(): + items.append(self._parse_element()) + return items[0] if len(items) == 1 else Seq(tuple(items)) + + def _parse_element(self) -> object: + tok = self._peek() + if tok is None: + raise PargParseError("unexpected end of input inside sequence") + if tok[0] == "num": + self._advance() + mn = int(tok[1]) + nxt = self._peek() + if nxt is not None and nxt[0] == "punct" and nxt[1] == "*": + self._advance() + nxt = self._peek() + mx = int(self._advance()[1]) if nxt is not None and nxt[0] == "num" else None + return self._capture_postfix(Rep(self._parse_rep_body(), mn, mx)) + return self._capture_postfix(Rep(self._parse_rep_body(), mn, mn)) + if tok[0] == "punct" and tok[1] == "*": + self._advance() + nxt = self._peek() + mx = int(self._advance()[1]) if nxt is not None and nxt[0] == "num" else None + return self._capture_postfix(Rep(self._parse_rep_body(), 0, mx)) + return self._parse_postfixed() + + def _parse_rep_body(self) -> object: + """A repetition's body is a guarded primary; it takes no `as` postfix.""" + sign, node = self._parse_guard_sign() + if sign is None: + return node + return Pred(sign, node) + + def _parse_guard_sign(self) -> tuple[bool | None, object]: + tok = self._peek() + if tok is not None and tok[0] == "punct" and tok[1] in ("!", "&"): + self._advance() + return (tok[1] == "&", self._parse_primary()) + return (None, self._parse_primary()) + + def _capture_postfix(self, node: object) -> object: + tok = self._peek() + if tok is not None and tok[0] == "ident" and tok[1] == "as": + self._advance() + name = self._expect("ident")[1] + return Cap(name, node) + return node + + def _parse_postfixed(self) -> object: + sign, node = self._parse_guard_sign() + if sign is None: + return self._capture_postfix(node) + # The capture binds the guard, not the guarded primary (parsanol-ruby#380). + return self._capture_postfix(Pred(sign, node)) + + def _parse_primary(self) -> object: + tok = self._advance() + kind, value, offset = tok + if kind == "str": + return Lit(_unquote(value)) + if kind == "sstr": + return Lit(value) + if kind == "hex": + return self._parse_hex(value) + if kind == "punct": + if value == "(": + node = self._parse_choice() + self._expect("punct", ")") + return node + if value == "[": + node = self._parse_choice() + self._expect("punct", "]") + return Rep(node, 0, 1, maybe=True) + raise PargParseError(f"unexpected {value!r} in expression at offset {offset}") + if kind == "ident": + if value == "until": + raise PargParseError(f'"until" is not supported by this reader (offset {offset})') + if value in ("alt", "state", "set", "switch"): + raise PargParseError(f'"{value}" is not supported by this reader (offset {offset})') + return Ref(value) + raise PargParseError(f"unexpected token {kind}:{value!r} at offset {offset}") + + def _parse_hex(self, spec: str) -> object: + """`%xNN-NN` range, `%xNN.NN` sequence of single-char classes, `%xNN` single.""" + if "-" in spec: + lo, hi = spec.split("-", 1) + if not lo or not hi: + raise PargParseError(f"malformed hex range %x{spec}") + return CharClass(((int(lo, 16), int(hi, 16)),)) + if "." in spec: + parts = tuple(CharClass(((int(h, 16), int(h, 16)),)) for h in spec.split(".")) + return parts[0] if len(parts) == 1 else Seq(parts) + code = int(spec, 16) + return CharClass(((code, code),)) + + +def parse_parg(text: str) -> PargGrammar: + """Read a PARG text into a grammar table.""" + return _PargParser(text).parse() diff --git a/interscript-py/src/interscript/grammar/peg.py b/interscript-py/src/interscript/grammar/peg.py new file mode 100644 index 00000000..0fd95f87 --- /dev/null +++ b/interscript-py/src/interscript/grammar/peg.py @@ -0,0 +1,460 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""PEG interpreter for grammar tables produced by parg_reader.py. + +Reproduces the parsanol (parslet-lineage) execution and tree-building +semantics exactly (mirroring interscript-ts's src/isc/grammar/peg.ts): + + - PEG ordered choice with full backtracking ACROSS alternatives, but + possessive repetition (a failed iteration never unwinds earlier ones) + - zero-width lookaheads (`!x` negative, `&x` positive) + - values are: Slice (matched text + offset), capture dict, list, None + - flattening (CanFlatten): a sequence folds its parts — string-likes + coalesce, a string-like meeting a hash/list is dropped, hashes merge + (later keys win), lists concat; a repetition containing hashes keeps + only the hashes (bare strings are discarded), one containing lists + flattens the lists, a purely textual one joins into one Slice; an + unnamed optional that matched nothing yields "" + +The result is the parslet-shaped capture tree the interscript-ruby +artifact parser produces, which the action layer (actions.py) then turns +into an IscDocument. + +Two additions over the ts interpreter, both needed for hand-parser +parity (see isc_parser.py): + + - SPAN RECORDING: Cap nodes whose name is in `span_names` record + (name, start, end) of their successful match, in parse order. The + compact rule's from/to atoms carry no delimiters in the capture + tree (closing quotes/brackets are not captured), but the W2 + override key is the byte-exact source text of the from atom — so + the interpreter reports the exact match spans and the action layer + pairs them with the tree in walk order. Memoized rule applications + store and replay their span delta, so memoization stays sound. + + - FAILURE DIAGNOSTICS: the interpreter tracks the furthest position + reached, the rule names that failed there, and every rule name that + failed anywhere, so isc_parser.py can render the two hand-parser + error messages the suite pins ("Unterminated string", keyword-in- + item-position) from real parse state instead of string matching. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .parg_reader import Alt, Cap, CharClass, Lit, Pred, Ref, Rep, Seq + + +@dataclass(frozen=True, slots=True) +class Slice: + """A matched span of input (parsanol Slice: offset + content).""" + + offset: int + content: str + + +class PegError(Exception): + """Raised when the root rule fails to consume the whole input.""" + + def __init__(self, message: str, at: int) -> None: + super().__init__(message) + self.at = at + + +# --------------------------------------------------------------------------- +# Flattening — direct port of Parsanol::CanFlatten. +# --------------------------------------------------------------------------- + + +def flatten(value: object, named: bool = False) -> object: + if not isinstance(value, list): + return value + tag = value[0] + + if len(value) == 2: + f = flatten(value[1], False) + if tag == "seq": + return f + if tag == "maybe": + # JS `f || ""`: only null/"" collapse; hashes and lists pass + # through (they are always truthy in JS). + return f if named else (f if not (f is None or f == "") else "") + if tag == "rep": + return _flatten_repetition([f], named) + + n = len(value) - 1 + result = [flatten(value[i + 1], False) for i in range(n)] + + if tag == "seq": + return _flatten_sequence(result) + if tag == "maybe": + first = result[0] if result else None + return first if named else (first if not (first is None or first == "") else "") + if tag == "rep": + return _flatten_repetition(result, named) + raise ValueError(f"BUG: unknown flatten tag {tag!r}") + + +def _flatten_sequence(lst: list) -> object: + # foldl: skip leading nils, merge the rest left-to-right. + i = 0 + n = len(lst) + while i < n and lst[i] is None: + i += 1 + if i >= n: + return "" + acc = lst[i] + i += 1 + while i < n: + e = lst[i] + if e is not None: + acc = _merge_fold(acc, e) + i += 1 + return acc + + +def _is_hash(v: object) -> bool: + return isinstance(v, dict) + + +def _is_stringlike(v: object) -> bool: + return isinstance(v, (str, Slice)) + + +def _slice_content(v: object) -> str: + return v.content if isinstance(v, Slice) else v # type: ignore[return-value,union-attr] + + +def _merge_fold(l: object, r: object) -> object: + l_hash = _is_hash(l) + r_hash = _is_hash(r) + l_str = _is_stringlike(l) + r_str = _is_stringlike(r) + + # equal pairs: merge + if l_hash and r_hash: + return {**l, **r} # type: ignore[dict-item,union-attr] + if isinstance(l, list) and isinstance(r, list): + return [*l, *r] + if isinstance(l, Slice) and isinstance(r, Slice): + return Slice(l.offset, l.content + r.content) + if isinstance(l, str) and isinstance(r, str): + return l + r + if l_str and r_str: + # Mixing a plain string with a Slice keeps the slice (its offset). + return r if isinstance(r, Slice) else l + # unequal pairs: a string-like meeting a structured value drops itself + if r_str: + return l + if l_str: + return r + # hash/list hoisting + if _is_hash(r): + return [l, r] + if _is_hash(l): + return [l, *r] # type: ignore[misc] + raise ValueError(f"BUG: unhandled merge_fold case ({type(l).__name__} / {type(r).__name__})") + + +def _flatten_repetition(lst: list, named: bool) -> object: + has_hash = False + has_array = False + for e in lst: + if _is_hash(e): + has_hash = True + elif isinstance(e, list): + has_array = True + if has_hash: + return [e for e in lst if _is_hash(e)] + if has_array: + flat: list = [] + for e in lst: + if isinstance(e, list): + flat.extend(e) + return flat + if named and not lst: + return [] + # All-string repetition: join into one slice. + first: Slice | None = None + content: str | None = None + for e in lst: + if e is None: + continue + if first is None and isinstance(e, Slice): + first = e + piece = _slice_content(e) + content = piece if content is None else content + piece + if content is None: + return "" + if first is not None: + return Slice(first.offset, content) + return content + + +# --------------------------------------------------------------------------- +# Interpreter +# --------------------------------------------------------------------------- + + +def _has_zero_named_charclass_rep(rules: dict) -> bool: + """Shape guard for the Rep(CharClass) scan fast path. The fast path + returns the repetition's FLATTENED value directly (a joined Slice, or + "" when zero iterations), which is identical to the slow path's + flatten EXCEPT for a zero-capable repetition sitting DIRECTLY under a + named capture (flatten yields [] there, not ""). No such shape exists + in the canonical ISC grammar; if a future grammar edit introduces + one, the guard disables the fast path for the whole table instead of + silently changing tree shapes.""" + + def walk(node: object) -> bool: + if isinstance(node, Cap): + inner = node.a + if ( + isinstance(inner, Rep) + and not inner.maybe + and inner.min == 0 + and isinstance(inner.a, CharClass) + ): + return True + return walk(inner) + if isinstance(node, (Seq, Alt)): + return any(walk(child) for child in node.a) + if isinstance(node, Pred): + return walk(node.a) + return False + + return any(walk(node) for node in rules.values()) + + +class PegInterpreter: + MAX_DEPTH = 200 + + def __init__( + self, + rules: dict, + src: str, + span_names: frozenset[str] = frozenset(), + ) -> None: + self._rules = rules + self._src = src + self._span_names = span_names + # Per-rule memo tables (rule order assigned lazily; pos -> result + # tuple). Memo values: (ok, value, end_pos, spans_delta, fail_name). + self._rule_ix: dict[int, int] = {} + self._memo: list[dict[int, tuple]] = [] + self.farthest = 0 + self.depth = 0 + # (name, start, end) of successful span-tracked captures, parse order. + self.spans: list[tuple[str, int, int]] = [] + # Rule names that failed AT the furthest position, and everywhere; + # plus every failure position per rule name (bounded by the input + # size) — the raw state the hand-parity error diagnostics read. + self.failed_at_farthest: set[str] = set() + self.failed_rules: set[str] = set() + self.failed_positions: dict[str, list[int]] = {} + # Rep(CharClass) scan fast path (see the Rep branch), disabled by + # the shape guard when a tree-shape-changing pattern exists. + self._fast_charclass_rep = not _has_zero_named_charclass_rep(rules) + + # -- entry ------------------------------------------------------------ + + def parse_root(self, root: str, start: int = 0) -> tuple[object, int]: + """Parse from `start`, requiring the ROOT rule plus full input + consumption. Returns (flattened capture tree, end position).""" + r = self._apply_rule(root, start) + ok, value, pos = r[0], r[1], r[2] + if not ok or pos != len(self._src): + at = pos if ok else self.farthest + raise PegError(self._describe_failure(root, at), at) + return flatten(value), pos + + def _describe_failure(self, root: str, at: int) -> str: + line = self._src.count("\n", 0, at) + 1 + col = at - self._src.rfind("\n", 0, at) + return f"Failed to match {root} at line {line} char {col}" + + # -- bookkeeping -------------------------------------------------------- + + def _note(self, pos: int) -> None: + self.farthest = max(self.farthest, pos) + + def _note_failure(self, pos: int, name: str) -> None: + self.failed_rules.add(name) + positions = self.failed_positions.setdefault(name, []) + if not positions or positions[-1] != pos: + positions.append(pos) + if pos > self.farthest: + self.farthest = pos + self.failed_at_farthest.clear() + if pos == self.farthest: + self.failed_at_farthest.add(name) + + # -- rules ---------------------------------------------------------------- + + def _apply_rule(self, name: str, pos: int) -> tuple: + ix = self._rule_ix.get(name) + if ix is None: + ix = self._rule_ix[name] = len(self._memo) + self._memo.append({}) + table = self._memo[ix] + hit = table.get(pos) + if hit is not None: + ok, value, r_pos, spans_delta, fail_name = hit + if ok: + if spans_delta: + self.spans.extend(spans_delta) + else: + self._note_failure(pos, fail_name) + return (ok, value, r_pos) + node = self._rules.get(name) + if node is None: + raise PegError(f"unknown rule {name!r}", pos) + self.depth += 1 + if self.depth > self.MAX_DEPTH: + self.depth -= 1 + raise PegError(f"recursion limit exceeded in rule {name!r}", pos) + mark = len(self.spans) + r = self._apply(node, pos, False) + self.depth -= 1 + if r[0]: + delta = tuple(self.spans[mark:]) if len(self.spans) != mark else () + memo = (True, r[1], r[2], delta, "") + else: + self._note_failure(pos, name) + del self.spans[mark:] + memo = (False, None, pos, (), name) + table[pos] = memo + return r + + # -- nodes ---------------------------------------------------------------- + + def _apply(self, node: object, pos: int, consume_all: bool) -> tuple: + # Dispatch ordered by measured hotness (Seq/Lit/CharClass dominate + # the identifier/whitespace rules). + kind = type(node) + if kind is Seq: + cur = pos + parts: list = ["seq"] + children = node.a # type: ignore[union-attr] + mark = len(self.spans) + n = len(children) + for i, child in enumerate(children): + r = self._apply(child, cur, consume_all and i == n - 1) + if not r[0]: + if len(self.spans) != mark: + del self.spans[mark:] + return (False, None, pos) + parts.append(r[1]) + cur = r[2] + return (True, parts, cur) + if kind is Lit: + s = node.s # type: ignore[union-attr] + src = self._src + if src.startswith(s, pos): + end = pos + len(s) + self._note(end) + return (True, Slice(pos, s), end) + return (False, None, pos) + if kind is CharClass: + # Character classes match ONE code point (Python strings are + # sequences of code points; the ts interpreter matches UTF-16 + # units and relies on textual repetitions rejoining surrogate + # halves — same tree content either way). Every class in the + # ISC grammar is BMP except the catch-all `chr` (%x00-10FFFF). + src = self._src + if pos >= len(src): + return (False, None, pos) + unit = ord(src[pos]) + for lo, hi in node.ranges: # type: ignore[union-attr] + if lo <= unit <= hi: + self._note(pos + 1) + return (True, Slice(pos, src[pos]), pos + 1) + return (False, None, pos) + if kind is Ref: + return self._apply_rule(node.name, pos) # type: ignore[union-attr] + if kind is Rep: + mn = node.min # type: ignore[union-attr] + mx = node.max # type: ignore[union-attr] + body = node.a # type: ignore[union-attr] + if ( + self._fast_charclass_rep + and not node.maybe # type: ignore[union-attr] + and type(body) is CharClass + ): + # Scan fast path: a pure char-class repetition folds to the + # same flattened value the slow path produces (a joined + # Slice, or "" for zero iterations). Shape-guarded — see + # _has_zero_named_charclass_rep. + src = self._src + ranges = body.ranges + n = len(src) + cur = pos + while cur < n: + unit = ord(src[cur]) + matched = False + for lo, hi in ranges: + if lo <= unit <= hi: + matched = True + break + if not matched: + break + cur += 1 + count = cur - pos + if count < mn: + self._note(cur) + return (False, None, pos) + self._note(cur) + if count == 0: + return (True, "", cur) + return (True, Slice(pos, src[pos:cur]), cur) + tag = "maybe" if node.maybe else "rep" # type: ignore[union-attr] + items: list = [tag] + cur = pos + count = 0 + mark = len(self.spans) + while mx is None or count < mx: + mark_i = len(self.spans) + r = self._apply(body, cur, False) + if not r[0]: + if len(self.spans) != mark_i: + del self.spans[mark_i:] + break + items.append(r[1]) + cur = r[2] + count += 1 + if count < mn: + self._note(cur) + if len(self.spans) != mark: + del self.spans[mark:] + return (False, None, pos) + return (True, items, cur) + if kind is Alt: + for branch in node.a: # type: ignore[union-attr] + mark = len(self.spans) + r = self._apply(branch, pos, consume_all) + if r[0]: + return r + if len(self.spans) != mark: + del self.spans[mark:] + return (False, None, pos) + if kind is Cap: + mark = len(self.spans) + r = self._apply(node.a, pos, consume_all) # type: ignore[union-attr] + if not r[0]: + if len(self.spans) != mark: + del self.spans[mark:] + return (False, None, pos) + if node.name in self._span_names: # type: ignore[union-attr] + self.spans.append((node.name, pos, r[2])) # type: ignore[union-attr] + return (True, {node.name: flatten(r[1], True)}, r[2]) # type: ignore[union-attr] + if kind is Pred: + mark = len(self.spans) + r = self._apply(node.a, pos, False) # type: ignore[union-attr] + if len(self.spans) != mark: + del self.spans[mark:] # zero-width: no capture survives a lookahead + matched = r[0] + if (node.positive and not matched) or (not node.positive and matched): # type: ignore[union-attr] + return (False, None, pos) + # Zero-width: position unchanged, value None. + return (True, None, pos) + raise TypeError(f"BUG: unknown PegNode {node!r}") diff --git a/interscript-py/src/interscript/isc.py b/interscript-py/src/interscript/isc.py index 8bb91150..c8189e89 100644 --- a/interscript-py/src/interscript/isc.py +++ b/interscript-py/src/interscript/isc.py @@ -1,8 +1,22 @@ # Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). -"""ISC parser — recursive-descent parser for the `.isc` map DSL. +"""ISC parser — public entry + document model + W1/W2 expansion. -Port of interscript-ts's `src/isc/parser.ts` (which mirrors the Ruby -Parslet grammar), extended with the ISC-WF-1 workflow constructs: +Parsing is GENERATED FROM THE GRAMMAR: the canonical `.parg` text +(vendored in `grammar/grammar_source.py`, sha256-pinned, synced from +interscript-ruby's lib/interscript/isc/grammar/isc.parg) is compiled by +a PARG reader (`grammar/parg_reader.py`), executed by a table-driven +PEG interpreter reproducing parsanol's capture/flattening semantics +(`grammar/peg.py`), and lowered to an IscDocument by the action layer +(`grammar/actions.py`) — mirroring interscript-ts's WO17 step-4b +generated parser. `parse_isc` keeps its public signature: + + doc = parse_isc(source, "map.isc") + doc = parse_isc(source, resolve_dependency=lambda code: other_doc) + cm = isc_to_compiled_map(doc) # -> executor.execute_stage(cm, ...) + +This module keeps what is NOT generated from the grammar — the ISC-WF-1 +document model and its runtime semantics (the ts runtime's +src/isc/types.ts + expand.ts equivalent): - **W1 named rulesets** — `ruleset { ... }` at map top level. - **W2 use with override** — `use [override { }]`, @@ -16,93 +30,57 @@ W1/W2 therefore never reach the executor: expanded maps compile to the same IR shapes as before, plus the two optional W3 fields. The parser -emits `_ir_types` structures directly, so the parse path and the JSON-IR -loader share one runtime representation. - - doc = parse_isc(source, "map.isc") - doc = parse_isc(source, resolve_dependency=lambda code: other_doc) - cm = isc_to_compiled_map(doc) # -> executor.execute_stage(cm, ...) - -Parser fidelity notes (mirroring the TypeScript reference): - -- Unrecognized tokens in stage/block bodies are silently dropped to end - of line (Ruby Parslet parity), except the new keywords, which parse. -- `when` joins the keyword table (§2): it is never consumed as a bare - alias reference in item position. +emits `_ir_types` structures directly, so the parse path and the +JSON-IR loader share one runtime representation. + +Parser fidelity notes: + +- The generated parser reproduces the former hand parser's documents + byte-for-byte across the whole maps corpus (434/434 through the + py-vs-py parity driver, scripts/parse_parity_py.py), including the + §4.2.6 trailing-parallel override join and byte-exact `from_source` + override keys. Documented hand-parser quirks the grammar rejects + (invalid `\\u` escapes, hyphenated identifiers outside run targets, + multi-digit capture refs, stage-level reverse/strip/swapcase) are + corpus-dormant; see grammar/actions.py for the full deviation list. - `languages` is declared once per map; a second declaration is a parse error. `use` may appear in stage bodies, ruleset bodies and inside `parallel`/`sequence` blocks; a ruleset whose expansion contains a pass block may not itself be used inside a pass block (a nested pass has no meaning in the one-simultaneous-pass model). + +Performance note (measured, same machine): the generated parser parses +the full 434-file corpus in ~86s where the hand parser took ~7s — the +interpreter spends its time in the same table-driven way the ts +generated parser does, and python function-call costs dominate. Typical +maps (5-50KB) parse in tens of milliseconds; the gap only shows on the +megabyte jamo/hanja maps, in the golden/embedded gates that parse the +whole corpus. Follow-ups (closure compilation, grammar-specific +prefilters) can close the gap without touching semantics; parity, not +speed, was this milestone's gate. """ from __future__ import annotations -import re from collections.abc import Callable from dataclasses import dataclass from typing import Any from ._ir_types import ( AliasItem, - AnyCharClassItem, - AnyItem, - CaptureGroupItem, - CaptureRefItem, CompiledMap, FuncallInline, FuncallRule, - GroupItem, ParallelRule, - RepeatItem, RunRule, SequentialRule, Stage, - StringItem, SubRule, ) from .errors import IscParseError, MapFormatError -PRIMITIVES = frozenset( - ["boundary", "line_start", "line_end", "word_boundary", "space", "non_boundary"] -) - -FUNCTIONS = frozenset( - ["upcase", "downcase", "title_case", "reverse", "strip", "swapcase"] -) - -CONSTRAINTS = frozenset(["before", "after", "not_before", "not_after"]) - -# ISC-WF-1 §2: the four new keywords join the keyword table. `use` / -# `override` / `languages` / `ruleset` are handled at their grammar -# positions; `when` can otherwise be reached from item position (right -# after a compact rule's `to`), so it is excluded from alias references. -ITEM_KEYWORDS = frozenset(["when"]) - -# Words that terminate a comma-less `ident (","? ident)*` list (the -# `languages` declaration and `when` guards). Without this, a one-line -# `languages kur stage main { }` would swallow `stage` as a language. -RESERVED_WORDS = frozenset( - [ - "metadata", "tests", "aliases", "stage", "dependency", "languages", - "ruleset", "sub", "use", "run", "parallel", "sequence", "separate", - "compose", "decompose", "when", "override", "note", "description", - "notes", "from", "to", "before", "after", "not_before", "not_after", - "as", "map", - ] -) - DependencyResolver = Callable[[str], "IscDocument | None"] -# JavaScript \s (the TypeScript parser's whitespace class), kept explicit -# so Python's Unicode \s cannot diverge from the reference. -_WS = frozenset("\t\n\v\f\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff") -_WORD = "A-Za-z0-9_" - -_RE_WORD = re.compile(f"[{_WORD}]") -_RE_IDENT = re.compile(f"[{_WORD}-]") -_RE_NUMBER = re.compile("[0-9]") - # --------------------------------------------------------------------------- # Parse-level document types (mirror interscript-ts src/isc/types.ts) @@ -183,805 +161,63 @@ class RawUse: # --------------------------------------------------------------------------- -# Parser +# W1/W2 expansion (unchanged from the replaced hand parser's _Parser; +# the generated action layer drives it after the capture tree is built) # --------------------------------------------------------------------------- -def parse_isc( - source: str, - filename: str | None = None, - resolve_dependency: DependencyResolver | None = None, -) -> IscDocument: - """Parse one .isc source into an expanded IscDocument. - - `resolve_dependency` maps a dependency's target system code to its - parsed IscDocument; it is called lazily when a `use alias.` - expansion needs the dependency's rulesets. W1 `use` splicing and W2 - override merging happen here, at parse time. - """ - return _Parser(source, filename, resolve_dependency).parse() - +class _Expander: + """Expands ruleset `use` directives (W1) and applies override rows + (W2, §4.2 + §4.2.6) over an already-parsed raw map body.""" -class _Parser: def __init__( self, + system_code: str, + dependencies: list[IscDependency], + resolve_dependency: DependencyResolver | None, src: str, filename: str | None, - resolve_dependency: DependencyResolver | None, + raw_rulesets: dict[str, list[Any]], ) -> None: - self.src = src - self.pos = 0 - self.filename = filename + self._system_code = system_code + self._dependencies = dependencies self.resolve_dependency = resolve_dependency - # Raw (unexpanded) map body, filled by parse(). - self._raw_rulesets: dict[str, list[Any]] = {} - self._dependencies: list[IscDependency] = [] - self._system_code = "" + self._src = src + self._filename = filename + # Raw (unexpanded) ruleset bodies, from the parsed tree. + self._raw_rulesets = raw_rulesets # Memo of fully expanded rulesets (this map only; dependency # rulesets arrive already expanded in their own IscDocument). - self._expanded: dict[str, list[Any]] = {} - - # -- Document --------------------------------------------------------- - - def parse(self) -> IscDocument: - self.skip_ws() - self.expect("system") - self.skip_ws() - self._system_code = self.parse_quoted_string() - self.skip_ws() - self.expect("{") - body = self.parse_block_items() - self.skip_ws() - self.expect("}") - self.skip_ws() - - metadata: dict[str, Any] = {} - tests: list[IscTest] = [] - aliases: list[tuple[str, Any]] = [] - raw_stages: list[tuple[str, list[Any]]] = [] - languages: list[str] | None = None - - for item in body: - tag = item[0] - if tag == "metadata": - metadata.update(item[1]) - elif tag == "tests": - tests = item[1] - elif tag == "aliases": - aliases = item[1] - elif tag == "stage": - raw_stages.append((item[1], item[2])) - elif tag == "dependency": - self._dependencies.append(item[1]) - elif tag == "languages": - if languages is not None: - self.error("languages declared more than once") - languages = item[1] - - # W1/W2 expansion pass (post-parse: forward references allowed). - for name in self._raw_rulesets: - self._expand_ruleset(name) - stages = [ - IscStage(name=name, items=tuple(self._expand_items(items, (), False))) - for name, items in raw_stages - ] - - return IscDocument( - system_code=self._system_code, - metadata=metadata, - tests=tuple(tests), - aliases=tuple(aliases), - stages=tuple(stages), - dependencies=tuple(self._dependencies), - languages=tuple(languages) if languages is not None else None, - rulesets={name: tuple(self._expanded[name]) for name in self._raw_rulesets}, - ) - - # -- Block items ------------------------------------------------------ - - def parse_block_items(self) -> list[Any]: - items: list[Any] = [] - while True: - self.skip_ws() - if self.peek() == "}" or self.pos >= len(self.src): - break - - keyword = self.peek_word() - if keyword == "metadata": - items.append(self.parse_metadata()) - elif keyword == "tests": - items.append(self.parse_tests()) - elif keyword == "aliases": - items.append(self.parse_aliases()) - elif keyword == "stage": - items.append(self.parse_stage()) - elif keyword == "dependency": - items.append(self.parse_dependency()) - elif keyword == "languages": - items.append(self.parse_languages()) - elif keyword == "ruleset": - items.append(self.parse_ruleset()) - else: - self.error(f"Unexpected keyword: {keyword}") - return items - - # -- Metadata --------------------------------------------------------- - - def parse_metadata(self) -> Any: - self.expect("metadata") - self.skip_ws() - self.expect("{") - metadata: dict[str, Any] = {} - - while True: - self.skip_ws() - if self.peek() == "}": - break - - if self.peek_word() == "description": - self.consume("description") - self.skip_inline_ws() - if self.peek() == "{": - self.expect("{") - metadata["description"] = self.parse_raw_text() - self.expect("}") - elif self.peek() == '"': - metadata["description"] = self.parse_quoted_string() - else: - metadata["description"] = self.parse_to_newline().strip() - continue - - if self.peek_word() == "notes": - self.consume("notes") - self.skip_inline_ws() - if self.peek() == "{": - self.expect("{") - # Note-list form (`note "..."`) vs raw free text. - saved = self.pos - self.skip_ws() - is_note_list = self.peek_word() == "note" - self.pos = saved - - if is_note_list: - notes: list[str] = [] - while True: - self.skip_ws() - if self.peek() == "}": - break - self.expect("note") - self.skip_ws() - notes.append(self.parse_quoted_string()) - self.expect("}") - metadata["notes"] = notes - else: - metadata["notes"] = self.parse_raw_text() - self.expect("}") - continue + self.expanded: dict[str, list[Any]] = {} - key = self.parse_identifier() - self.skip_inline_ws() - if self.peek() in ("\n", "}"): - metadata[key] = "" - continue - if self.peek() == "{": - self.expect("{") - metadata[key] = self.parse_raw_text() - self.expect("}") - elif self.peek() == '"': - metadata[key] = self.parse_quoted_string() - else: - metadata[key] = self.parse_to_newline().strip() - self.expect("}") - return ("metadata", metadata) - - # -- Tests ------------------------------------------------------------ - - def parse_tests(self) -> Any: - self.expect("tests") - self.skip_ws() - self.expect("{") - tests: list[IscTest] = [] - - while True: - self.skip_ws() - if self.peek() == "}": - break - if self.peek() == "#": - self.skip_line() - continue - - src = self.parse_string_literal() - self.skip_ws() - self.expect("->") - self.skip_ws() - expected = self.parse_string_literal() - self.skip_ws() - - if self.peek_word() == "note": - self.consume("note") - self.skip_ws() - tests.append(IscTest(input=src, expected=expected, note=self.parse_string_literal())) - else: - tests.append(IscTest(input=src, expected=expected)) - self.expect("}") - return ("tests", tests) - - # -- Aliases ---------------------------------------------------------- - - def parse_aliases(self) -> Any: - self.expect("aliases") - self.skip_ws() - self.expect("{") - aliases: list[tuple[str, Any]] = [] - - while True: - self.skip_ws() - if self.peek() == "}": - break - if self.peek() == "#": - self.skip_line() - continue - - name = self.parse_identifier() - self.skip_ws() - self.expect("=") - self.skip_ws() - aliases.append((name, self.parse_item(False))) - self.expect("}") - return ("aliases", aliases) - - # -- Languages (ISC-WF-1 W3) ------------------------------------------ - - def parse_languages(self) -> Any: - self.expect("languages") - self.skip_ws() - names = self.parse_ident_list() - return ("languages", names) - - def parse_ident_list(self) -> list[str]: - """`ident (","? ident)*` — the shared list form of `languages` and - `when`. A comma always continues the list; a comma-less word - continues it only when it is not a reserved keyword, so a - one-line declaration cannot swallow the next construct.""" - names = [self.parse_identifier()] - while True: - self.skip_inline_ws() - if self.peek() == ",": - self.pos += 1 - self.skip_ws() - names.append(self.parse_identifier()) - continue - if _RE_IDENT.match(self.peek()) and self.peek_word() not in RESERVED_WORDS: - names.append(self.parse_identifier()) - continue - break - return names - - # -- Stage and ruleset bodies ----------------------------------------- - - def parse_stage(self) -> Any: - self.expect("stage") - self.skip_ws() - name = self.parse_identifier() - self.skip_ws() - self.expect("{") - body = self.parse_stage_items() - self.expect("}") - return ("stage", name, body) - - def parse_ruleset(self) -> Any: - """ISC-WF-1 W1: `ruleset { }`.""" - self.expect("ruleset") - self.skip_ws() - name = self.parse_identifier() - self.skip_ws() - self.expect("{") - body = self.parse_stage_items() - self.expect("}") - if name in self._raw_rulesets: - self.error(f"ruleset declared more than once: {name}") - self._raw_rulesets[name] = body - return ("ruleset", name) - - def parse_stage_items(self) -> list[Any]: - """Shared body grammar of `stage` and `ruleset` blocks.""" - body: list[Any] = [] - while True: - self.skip_ws() - if self.peek() == "}": - break - if self.peek() == "#": - self.skip_line() - continue - - keyword = self.peek_word() - if keyword in ("parallel", "sequence"): - self.consume(keyword) - self.skip_ws() - self.expect("{") - rules: list[Any] = [] - while True: - self.skip_ws() - if self.peek() == "}": - break - if self.peek() == "#": - self.skip_line() - continue - if self.peek_word() == "sub": - rules.append(self.parse_rule()) - elif self.peek_word() == "use": - rules.append(self.parse_use()) - else: - # Ruby Parslet silently drops unrecognized tokens - # in rule blocks. - self.skip_line() - self.expect("}") - body.append(ParallelBlock(tuple(rules)) if keyword == "parallel" else SequenceBlock(tuple(rules))) - elif keyword == "run": - body.append(self.parse_run()) - elif keyword == "separate": - self.consume("separate") - self.skip_ws() - if self.peek_word() == "separator": - self.consume("separator") - self.skip_ws() - atom = self.parse_item_atom(False) - if isinstance(atom, StringItem): - body.append(FuncallRule(name="separate", kwargs={"separator": atom.value})) - else: - body.append(FuncallRule(name="separate")) - else: - body.append(FuncallRule(name="separate")) - elif keyword in ("compose", "decompose"): - self.consume(keyword) - body.append(FuncallRule(name=keyword)) - elif keyword == "rababa": - self.consume("rababa") - self.skip_inline_ws() - kwargs: dict[str, str] = {} - while self.peek() not in ("}", "", "\n"): - arg_name = self.parse_identifier() - self.skip_inline_ws() - self.expect(":") - self.skip_inline_ws() - kwargs[arg_name] = self.parse_string_literal() - self.skip_inline_ws() - if self.peek() == ",": - self.pos += 1 - self.skip_inline_ws() - body.append(FuncallRule(name="rababa", kwargs=kwargs)) - elif keyword in FUNCTIONS: - self.consume(keyword) - body.append(FuncallRule(name=keyword)) - elif keyword == "sub": - body.append(self.parse_rule()) - elif keyword == "use": - body.append(self.parse_use()) - else: - # Ruby Parslet silently drops unrecognized tokens in stage - # body. Skip to end of line for parity. - self.skip_line() - return body - - def parse_run(self) -> RunItem: - self.consume("run") - self.skip_ws() - dep: str | None = None - if self.peek_word() == "map": - self.consume("map") - self.expect(".") - dep = self.parse_identifier() - self.expect(".") - self.expect("stage") - self.expect(".") - stage_name = self.parse_identifier() - elif self.peek_word() == "stage": - self.consume("stage") - self.expect(".") - stage_name = self.parse_identifier() - else: - self.error("Expected map. or stage. after run") - return RunItem(dependency=dep, stage=stage_name) - - # -- Use (ISC-WF-1 W1/W2) ---------------------------------------------- - - def parse_use(self) -> RawUse: - self.expect("use") - self.skip_ws() - first = self.parse_identifier() - dependency_alias: str | None = None - self.skip_ws() - if self.peek() == ".": - self.pos += 1 - self.skip_ws() - dependency_alias = first - name = self.parse_identifier() - else: - name = first - - override: list[ParsedRule] | None = None - self.skip_ws() - if self.peek_word() == "override": - self.consume("override") - self.skip_ws() - self.expect("{") - override = [] - while True: - self.skip_ws() - if self.peek() == "}": - break - if self.peek() == "#": - self.skip_line() - continue - if self.peek_word() == "sub": - override.append(self.parse_rule()) - else: - self.skip_line() - self.expect("}") - return RawUse( - name=name, - dependency_alias=dependency_alias, - override=tuple(override) if override is not None else None, - ) - - # -- Rules -------------------------------------------------------------- - - def parse_rule(self) -> ParsedRule: - self.expect("sub") - self.skip_ws() - from_source: str | None = None - when: tuple[str, ...] | None = None - - # Block form: sub { from ... to ... } - if self.peek() == "{": - self.expect("{") - from_: Any = StringItem(value="") - to: Any = StringItem(value="") - constraints: list[tuple[str, Any]] = [] - - while True: - self.skip_ws() - if self.peek() == "}": - break - - word = self.peek_word() - if word == "from": - self.consume("from") - self.skip_ws() - from_ = self.parse_item(False) - elif word == "to": - self.consume("to") - self.skip_ws() - to = self.parse_item(True) - elif word in CONSTRAINTS: - self.consume(word) - self.skip_ws() - constraints.append((word, self.parse_item(False))) - else: - self.error(f"Unexpected keyword in sub block: {word}") - self.expect("}") - self.skip_ws() - if self.peek_word() == "when": - self.consume("when") - self.skip_ws() - when = tuple(self.parse_ident_list()) - return ParsedRule( - rule=_make_sub_rule(from_, to, constraints, when), - from_source=None, - ) - - # Compact form: sub [when ...] - start = self.pos - from_ = self.parse_item_atom(False) - from_source = self.src[start : self.pos].strip() - self.skip_ws() - to = self.parse_item_atom(True) - constraints: list[tuple[str, Any]] = [] - - while True: - self.skip_ws() - word = self.peek_word() - if word not in CONSTRAINTS: - break - self.consume(word) - self.skip_ws() - constraints.append((word, self.parse_item_atom(False))) - - self.skip_ws() - if self.peek_word() == "when": - self.consume("when") - self.skip_ws() - when = tuple(self.parse_ident_list()) - - return ParsedRule( - rule=_make_sub_rule(from_, to, constraints, when), - from_source=from_source, - ) + def error(self, msg: str) -> None: + pos = len(self._src) + line = self._src[:pos].count("\n") + 1 + col = pos - self._src.rfind("\n", 0, pos) + full_msg = f"{self._filename}: {msg}" if self._filename else msg + raise IscParseError(full_msg, pos, line, col) - # -- Items --------------------------------------------------------------- - - def parse_item(self, to_position: bool) -> Any: - first, first_kind = self.parse_item_atom_raw() - parts = [first] - kinds = [first_kind] - - while True: - self.skip_inline_ws() - if self.peek() == "+": - self.pos += 1 - self.skip_inline_ws() - atom, kind = self.parse_item_atom_raw() - parts.append(atom) - kinds.append(kind) - elif self._is_item_atom_start() and not self._is_keyword(): - atom, kind = self.parse_item_atom_raw() - parts.append(atom) - kinds.append(kind) - else: - break - - if len(parts) == 1: - return _function_to_inline(parts[0], kinds[0]) if to_position else parts[0] - return GroupItem(items=tuple(parts)) - - def parse_item_atom(self, to_position: bool) -> Any: - atom, kind = self.parse_item_atom_raw() - return _function_to_inline(atom, kind) if to_position else atom - - def parse_item_atom_raw(self) -> tuple[Any, str]: - """One item atom -> (IR item, parse kind). Function atoms keep - their kind so the `to` position can lower them to FuncallInline - (mirrors converter.ts); elsewhere they fall back to alias lookup - so a clear error surfaces at runtime.""" - c = self.peek() - - if c in ('"', "'"): - return StringItem(value=self.parse_string_literal()), "string" - - word = self.peek_word() - if not word: - self.error("Expected item atom") - - if word in ITEM_KEYWORDS: - self.error(f"Unexpected keyword in item position: {word}") - - if word == "none": - self.consume("none") - # converter.ts lowers `none` to the empty string. - return StringItem(value=""), "string" - if word == "any_character": - self.consume("any_character") - return AliasItem(name="any_character"), "function" - - if word == "any": - self.consume("any") - self.expect("(") - self.skip_ws() - inner = self.peek() - # any("...".."...") or any('...'..'...') — range from strings - if inner in ('"', "'"): - lo = self.parse_string_literal() - self.skip_ws() - if self.src.startswith("..", self.pos): - self.pos += 2 - self.skip_ws() - hi = self.parse_string_literal() - self.skip_ws() - self.expect(")") - return AnyCharClassItem(range=(lo, hi)), "item" - self.expect(")") - # any("abc") — split into per-char string items - return ( - AnyItem(of=tuple(StringItem(value=ch) for ch in lo)), - "item", - ) - if inner == "[": - self.expect("[") - items: list[Any] = [] - while True: - self.skip_ws() - if self.peek() == "]": - break - items.append(self.parse_item(False)) - self.skip_ws() - if self.peek() == ",": - self.pos += 1 - self.expect("]") - self.skip_ws() - self.expect(")") - return AnyItem(of=tuple(items)), "item" - # any(item), any(item + item), any(alias_name), any(range..) - item = self.parse_item(False) - self.skip_ws() - self.expect(")") - return AnyItem(of=(item,)), "item" - - if word == "capture": - self.consume("capture") - self.expect("(") - self.skip_ws() - inner = self.parse_item(False) - self.skip_ws() - self.expect(")") - return CaptureGroupItem(data=inner), "item" - - if word == "maybe": - self.consume("maybe") - self.expect("(") - self.skip_ws() - inner = self.parse_item(False) - self.skip_ws() - self.expect(")") - return RepeatItem(item=inner, min=0, max=1), "item" - - if word == "some": - self.consume("some") - self.expect("(") - self.skip_ws() - inner = self.parse_item(False) - self.skip_ws() - self.expect(")") - return RepeatItem(item=inner, min=1, max=None), "item" - - if word == "ref": - self.consume("ref") - self.expect("(") - self.skip_ws() - num = self.parse_number() - self.skip_ws() - self.expect(")") - return CaptureRefItem(id=num), "item" - - if word in PRIMITIVES: - self.consume(word) - return AliasItem(name=word), "item" - - if word in FUNCTIONS: - self.consume(word) - return AliasItem(name=word), "function" - - # Alias reference - self.consume(word) - return AliasItem(name=word), "item" - - # -- Dependency -------------------------------------------------------- - - def parse_dependency(self) -> Any: - self.expect("dependency") - self.skip_ws() - target = self.parse_quoted_string() - self.skip_inline_ws() - alias_name: str | None = None - if self.peek_word() == "as": - self.consume("as") - self.skip_ws() - alias_name = self.parse_identifier() - return ("dependency", IscDependency(target=target, alias_name=alias_name)) - - # -- String primitives --------------------------------------------------- - - def parse_string_literal(self) -> str: - c = self.peek() - if c == '"': - return self.parse_quoted_string() - if c == "'": - return self.parse_single_quoted_string() - self.error(f'Expected string but got "{self.src[self.pos : self.pos + 20]}"') - - def parse_quoted_string(self) -> str: - self.expect('"') - result: list[str] = [] - while self.pos < len(self.src): - c = self.src[self.pos] - self.pos += 1 - if c == '"': - return "".join(result) - if c == "\\": - if self.pos >= len(self.src): - break - nxt = self.src[self.pos] - self.pos += 1 - if nxt == "n": - result.append("\n") - elif nxt == "t": - result.append("\t") - elif nxt == "r": - result.append("\r") - elif nxt == '"': - result.append('"') - elif nxt == "\\": - result.append("\\") - elif nxt == "u": - hex_ = self.src[self.pos : self.pos + 4] - self.pos += 4 - try: - result.append(chr(int(hex_, 16))) - except ValueError: - result.append("\0") - elif nxt == "{": - result.append("{") - elif nxt == "}": - result.append("}") - else: - result.append(nxt) - else: - result.append(c) - self.error("Unterminated string") - - def parse_single_quoted_string(self) -> str: - self.expect("'") - start = self.pos - while self.pos < len(self.src) and self.src[self.pos] != "'": - self.pos += 1 - result = self.src[start : self.pos] - self.expect("'") - return result - - def parse_raw_text(self) -> str: - result: list[str] = [] - depth = 0 - while self.pos < len(self.src): - c = self.src[self.pos] - if c == "\\" and self.pos + 1 < len(self.src): - nxt = self.src[self.pos + 1] - if nxt in "{}": - result.append(nxt) - self.pos += 2 - continue - if c == "}" and depth == 0: - break - if c == "{": - depth += 1 - elif c == "}": - depth -= 1 - result.append(c) - self.pos += 1 - return "".join(result).strip() - - def parse_identifier(self) -> str: - self.skip_ws() - start = self.pos - while self.pos < len(self.src) and _RE_IDENT.match(self.src[self.pos]): - self.pos += 1 - if self.pos == start: - self.error("Expected identifier") - return self.src[start : self.pos] - - def parse_number(self) -> int: - start = self.pos - while self.pos < len(self.src) and _RE_NUMBER.match(self.src[self.pos]): - self.pos += 1 - return int(self.src[start : self.pos] or "0", 10) - - def parse_to_newline(self) -> str: - start = self.pos - while self.pos < len(self.src) and self.src[self.pos] != "\n": - self.pos += 1 - return self.src[start : self.pos] - - # -- W1/W2 expansion ----------------------------------------------------- - - def _expand_ruleset(self, name: str, chain: tuple[tuple[str, str], ...] = ()) -> list[Any]: + def expand_ruleset(self, name: str, chain: tuple[tuple[str, str], ...] = ()) -> list[Any]: entry = (self._system_code, name) if entry in chain: self.error(f"use cycle: {' -> '.join(f'{m}/{r}' for m, r in (*chain, entry))}") - cached = self._expanded.get(name) + cached = self.expanded.get(name) if cached is not None: return cached raw = self._raw_rulesets.get(name) if raw is None: self.error(f"unknown ruleset: {name}") - expanded = self._expand_items(raw, chain + (entry,), False) - self._expanded[name] = expanded + expanded = self.expand_items(raw, chain + (entry,), False) + self.expanded[name] = expanded return expanded - def _expand_items( + def expand_items( self, items: list[Any], chain: tuple[tuple[str, str], ...], in_block: bool ) -> list[Any]: out: list[Any] = [] for it in items: if isinstance(it, RawUse): - out.extend(self._expand_use(it, chain, in_block)) + out.extend(self.expand_use(it, chain, in_block)) continue if in_block and isinstance(it, (ParallelBlock, SequenceBlock)): self.error( @@ -989,17 +225,17 @@ def _expand_items( "used inside a parallel/sequence block" ) if isinstance(it, (ParallelBlock, SequenceBlock)): - it = type(it)(rules=tuple(self._expand_items(list(it.rules), chain, True))) + it = type(it)(rules=tuple(self.expand_items(list(it.rules), chain, True))) out.append(it) return out - def _expand_use( + def expand_use( self, use: RawUse, chain: tuple[tuple[str, str], ...], in_block: bool ) -> list[Any]: if use.dependency_alias is not None: - expanded = self._expand_via_dependency(use, chain) + expanded = self.expand_via_dependency(use, chain) else: - expanded = self._expand_ruleset(use.name, chain) + expanded = self.expand_ruleset(use.name, chain) if in_block and any(isinstance(x, (ParallelBlock, SequenceBlock)) for x in expanded): self.error( @@ -1011,7 +247,7 @@ def _expand_use( expanded = self._apply_override(expanded, list(use.override)) return list(expanded) - def _expand_via_dependency( + def expand_via_dependency( self, use: RawUse, chain: tuple[tuple[str, str], ...] ) -> list[Any]: target = next( @@ -1057,8 +293,8 @@ def _apply_override( pass, §4.2.5/§4.2.6); otherwise O appends after R_kept as stage-level rows. Cross-grammar canonical: the ruby reference and the ts runtime both join the trailing parallel (found by - maps scripts/parse-parity.mjs — py appended unmatched additions - as bare stage rules, splitting the pass).""" + maps scripts/parse-parity.mjs — pre-parity py appended unmatched + additions as bare stage rules, splitting the pass).""" okeys = [self._from_key(o) for o in overrides] out: list[Any] = [] for it in items: @@ -1082,69 +318,27 @@ def _apply_override( out.extend(overrides) return out - # -- Scanner helpers (mirror the TypeScript parser) ---------------------- - - def char_at(self, i: int) -> str: - return self.src[i] if i < len(self.src) else "" - - def peek(self) -> str: - return self.char_at(self.pos) - - def peek_word(self) -> str: - start = self.pos - while self.pos < len(self.src) and _RE_WORD.match(self.src[self.pos]): - self.pos += 1 - word = self.src[start : self.pos] - self.pos = start - return word - - def expect(self, s: str) -> None: - self.skip_ws() - if not self.src.startswith(s, self.pos): - self.error(f'Expected "{s}" but got "{self.src[self.pos : self.pos + 20]}"') - self.pos += len(s) - - def consume(self, word: str) -> None: - if not self.src.startswith(word, self.pos): - self.error(f'Expected "{word}"') - self.pos += len(word) - - def skip_ws(self) -> None: - while self.pos < len(self.src): - c = self.src[self.pos] - if c in _WS: - self.pos += 1 - continue - if c == "#": - self.skip_line() - continue - break - def skip_inline_ws(self) -> None: - while self.pos < len(self.src) and self.src[self.pos] in " \t": - self.pos += 1 +# --------------------------------------------------------------------------- +# Public parse entry — generated parser (grammar/) +# --------------------------------------------------------------------------- - def skip_line(self) -> None: - while self.pos < len(self.src) and self.src[self.pos] != "\n": - self.pos += 1 - if self.pos < len(self.src): - self.pos += 1 - def _is_item_atom_start(self) -> bool: - c = self.peek() - if not c: - return False - return c in ('"', "'") or bool(_RE_WORD.match(c)) +def parse_isc( + source: str, + filename: str | None = None, + resolve_dependency: DependencyResolver | None = None, +) -> IscDocument: + """Parse one .isc source into an expanded IscDocument. - def _is_keyword(self) -> bool: - word = self.peek_word() - return word in ("to", "from", "before", "after", "not_before", "not_after", "note") + `resolve_dependency` maps a dependency's target system code to its + parsed IscDocument; it is called lazily when a `use alias.` + expansion needs the dependency's rulesets. W1 `use` splicing and W2 + override merging happen here, at parse time. + """ + from .grammar.isc_parser import parse_isc as _generated_parse - def error(self, msg: str) -> None: - line = self.src[: self.pos].count("\n") + 1 - col = self.pos - self.src.rfind("\n", 0, self.pos) - full_msg = f"{self.filename}: {msg}" if self.filename else msg - raise IscParseError(full_msg, self.pos, line, col) + return _generated_parse(source, filename, resolve_dependency) # --------------------------------------------------------------------------- diff --git a/interscript-py/tests/test_grammar_source.py b/interscript-py/tests/test_grammar_source.py new file mode 100644 index 00000000..ddfb6b6e --- /dev/null +++ b/interscript-py/tests/test_grammar_source.py @@ -0,0 +1,68 @@ +# Copyright 2026 Interscript Project. BSD-3-Clause (see LICENSE). +"""Grammar provenance pin — the embedded ISC grammar text must match the +sha256 recorded at sync time. A mismatch means grammar_source.py was +edited by hand instead of via scripts/sync_isc_grammar.py, or the sync +was run against a different interscript-ruby revision than the pin +records. Either way the drift must be deliberate, never silent. + +(Mirror of interscript-ts's test/isc/grammar-source.test.ts — both +vendored copies must pin the same .parg text.) +""" + +import hashlib + +from interscript.grammar import isc_parser +from interscript.grammar.grammar_source import ISC_PARG_SHA256, ISC_PARG_SOURCE +from interscript.isc import parse_isc + + +class TestGrammarSourcePin: + def test_matches_recorded_sha256(self): + sha = hashlib.sha256(ISC_PARG_SOURCE.encode("utf-8")).hexdigest() + assert sha == ISC_PARG_SHA256 + + def test_declares_isc_entry_point(self): + # A cheap structural sanity check alongside the hash pin. + assert "entry isc: isc_source" in ISC_PARG_SOURCE + assert 'grammar InterscriptIsc version "' in ISC_PARG_SOURCE + + def test_compiles_and_parses(self): + doc = parse_isc('system "pin-test" { stage main { } }', "pin.isc") + assert doc.system_code == "pin-test" + + def test_extension_targets_present_in_canonical_grammar(self): + # The shape guard in compile_grammar duplicates this; pin it + # independently so a renamed rule fails HERE with a clear message. + grammar = isc_parser.parse_parg(ISC_PARG_SOURCE) + for name in ("any_arg", "block_rule", "run_dep", "rule_items"): + assert name in grammar.rules, f"GRAMMAR_EXTENSIONS target {name} missing" + + +class TestGeneratedParserPins: + """The hand-parser behaviors the generated parser must keep (the + error-message shapes the suite relies on).""" + + def test_unterminated_string_message(self): + import pytest + + from interscript.errors import IscParseError + + with pytest.raises(IscParseError, match="Unterminated string"): + parse_isc('system "s { stage main { } }', "t.isc") + + def test_keyword_in_item_position_message(self): + import pytest + + from interscript.errors import IscParseError + + with pytest.raises(IscParseError, match="keyword"): + parse_isc('system "s" { stage main { sub when "y" } }', "t.isc") + + def test_error_position_line(self): + import pytest + + from interscript.errors import IscParseError + + with pytest.raises(IscParseError) as exc: + parse_isc('system "s" {\n bogus {\n}\n', "t.isc") + assert exc.value.line == 2