Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 8 additions & 14 deletions interscript-py/src/interscript/grammar/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,20 +241,14 @@ def string(self, tree: Tree) -> str:
out.append("U")
elif "unicode" in h:
digits = _text(h["unicode"]) # type: ignore[arg-type]
# DEVIATION (hand-parser parity): the grammar cannot
# distinguish `\uXXXX` from `\UXXXX` at the capture-name
# level (both are :unicode) — but the hand parser only
# DECODED lowercase \u and rendered a capital \U as a
# literal "U" followed by the raw digits. The slice offset
# tells us which spelling the source used (the escape
# letter sits one position before the digits). Mirroring
# the hand parser keeps documents byte-identical across
# the swap. (The ruby artifact parser decodes both
# spellings; aligning runtimes is an owner decision.)
if ord(self._src[h["unicode"].offset - 1]) == 0x55:
out.append("U" + digits)
else:
out.append(chr(int(digits, 16)))
# WO19 G6 ruling: BOTH escape spellings decode. The former
# hand parser only decoded lowercase \u and rendered a
# capital \U as a literal "U" followed by the raw digits,
# so gki-bel's `sub "\U040E" "Ŭ"` carried the from-value
# "U040E" and the row never fired. The grammar captures
# both spellings under :unicode and the tree carries the
# decoded codepoint (ruby/rs always decoded both).
out.append(chr(int(digits, 16)))
else:
raise ValueError(f"unknown string part {sorted(h.keys())}")
return "".join(out)
Expand Down
6 changes: 3 additions & 3 deletions interscript-py/src/interscript/grammar/grammar_source.py

Large diffs are not rendered by default.

28 changes: 13 additions & 15 deletions interscript-py/src/interscript/grammar/isc_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
Single-sourcing contract: the grammar TEXT embedded in grammar_source.py
is the semantic source of truth (synced from interscript-ruby's
lib/interscript/isc/grammar/isc.parg via scripts/sync_isc_grammar.py).
The three documented grammar-table extensions below (GRAMMAR_EXTENSIONS)
The two documented grammar-table extensions below (GRAMMAR_EXTENSIONS)
restore hand-parser behaviors the canonical .parg rejects — each is
spelled in the grammar language, named, and corpus-proven dormant.
"""
Expand Down Expand Up @@ -66,10 +66,10 @@ def compile_grammar() -> CompiledGrammar:

grammar = parse_parg(ISC_PARG_SOURCE)
rules = dict(grammar.rules)
# Shape-guarded merge: if a future .parg edit renames any of the three
# rules, the merge raises instead of silently misparsing.
# Shape-guarded merge: if a future .parg edit renames any of the
# extension rules, the merge raises instead of silently misparsing.
ext = parse_parg(GRAMMAR_EXTENSIONS)
for name in ("any_arg", "block_rule", "run_dep"):
for name in ("any_arg", "run_dep"):
if name not in rules or name not in ext.rules:
raise RuntimeError(
f"grammar extension target {name} missing from isc.parg — "
Expand Down Expand Up @@ -139,7 +139,6 @@ def _hand_style_failure(source, c, interp, peg_error) -> tuple[str, int]:
GRAMMAR_EXTENSIONS = """
grammar IscExtensions version "0.0.0" {
any_arg = range_arg / set_arg / alias_arg / ((item) as any_item)
block_rule = ("sub" [ws] "{" [ws] "from" whitespace ((item) as from) [ws] "to" whitespace ((item) as to) ((*(whitespace constraint)) as constraints) [ws] "}" [(when_guard)]) / ("sub" [ws] "{" [ws] "from" whitespace ((item) as from) [ws] "to" whitespace ((item) as to) ((*(whitespace constraint)) as constraints) [(when_guard)] [ws] "}")
run_dep = "map." (((identifier *("-" identifier)) / identifier) as dep) ".stage." (identifier as stage)
}
"""
Expand All @@ -148,30 +147,29 @@ def _hand_style_failure(source, c, interp, peg_error) -> tuple[str, int]:
# syntax the canonical .parg rejects. Each is spelled in the grammar
# language itself and merged into the rule table, so the .parg remains
# the only semantic source and every extension is one visible, named
# block (all three are the same extensions the ts generated parser
# ships):
# block (all remaining ones are the same extensions the ts generated
# parser ships):
#
# 1. `any_arg` — restore the ruby DSL's final `item as :any_item`
# fallback (concerns/items.rb any_constructor), which the canonical
# .parg dropped; the interscript-rs vendored copy keeps it.
# (`any(space)`, `any(space + line_end)` — the ts suite pins both.)
# 2. `block_rule` — accept the `when` guard AFTER the closing brace of
# the block form. The ruby DSL places the guard inside the braces;
# the ts and py runtimes both place it outside, and the py suite pins
# the outside form (test_isc_parser.py::test_when_suffix_after_block_form).
# The outside branch is tried first (it is the greedy one; its
# mandatory `}` keeps it from shadowing the inside form).
# 3. `run_dep` — allow hyphenated dependency names (`run map.some-dep.…`).
# 2. `run_dep` — allow hyphenated dependency names (`run map.some-dep.…`).
# The grammar's identifier is [A-Za-z_][A-Za-z0-9_]*; the former hand
# parser's identifier admitted `-` in structural positions. Only the
# dependency slot is widened; the corpus is hyphen-free in every
# identifier position (proven by the corpus parity driver).
# The former fourth extension (`rule_items` accepting `use` inside
# The former third extension (`rule_items` accepting `use` inside
# parallel/sequence blocks) was RETIRED by WO18 G5: the canonical .parg
# now carries use_directive in rule_line (spec §3.2, mirroring the rs
# grammar), so the extension is redundant and its merge-guard slot is
# gone. The override block's rule_line accepts `use` rows for free —
# the expansion flattens them (isc._Expander._expand_override_rows).
# The former fourth extension (block_rule accepting the `when` guard
# AFTER the closing brace) was RETIRED by WO19 item 5: the canonical
# binding is INSIDE the braces — rs (the golden executor) and the ruby
# DSL reject the outside form outright, and the generated grammar now
# matches them (test_isc_wf1.py::test_when_after_block_brace_rejected).
#
# Ruby rejects (1)'s absence aside, the outside-when and hyphen-dep
# forms are cross-runtime divergences (ruby parses none); aligning
Expand Down
82 changes: 82 additions & 0 deletions interscript-py/src/interscript/grammar/peg.py
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,14 @@ def rep(pos, I):

def c_pred(body, positive):
def pred(pos, I):
spans = I.spans
mark = len(spans)
r = body(pos, I)
# A lookahead never contributes captures: drop any match
# spans its body recorded, success or failure (WO19: without
# this, a speculative rule re-parse inside a lookahead leaks
# from/to spans and desyncs the action layer's span walk).
del spans[mark:]
if r[0] is not positive:
return _FAIL
return (True, None, pos)
Expand Down Expand Up @@ -846,6 +853,32 @@ def seq(pos, I):

return seq

def c_seq_final_span(children):
# c_seq_final with span rollback: when a child fails, spans any
# EARLIER sibling recorded must be dropped — the sequence never
# happened. (WO19: the bare variant let a failed speculative
# parse leak from/to spans, desyncing the action layer's span
# walk on block rules carrying a `when` guard.)
def seq(pos, I):
spans = I.spans
cur = pos
parts: list = []
mark = len(spans)
for f, is_raw in children:
r = f(cur, I)
if not r[0]:
if len(spans) != mark:
del spans[mark:]
return _FAIL
if is_raw:
parts.append(flatten(r[1], False))
else:
parts.append(r[1])
cur = r[2]
return (True, _flatten_sequence(parts), cur)

return seq

def c_rep_final(body, body_raw, mn, mx, maybe, named):
# The repetition's value is the CanFlatten fold over its
# flattened items (named flag: direct capture bodies fold an
Expand Down Expand Up @@ -881,6 +914,44 @@ def rep(pos, I):

return rep

def c_rep_final_span(body, body_raw, mn, mx, maybe, named):
# c_rep_final with span rollback: a failed iteration drops its
# own partial spans; a rep that fails outright (count < mn) drops
# every iteration's spans. (WO19, see c_seq_final_span.)
def rep(pos, I):
spans = I.spans
parts: list = []
cur = pos
count = 0
mark = len(spans)
while mx is None or count < mx:
mark_i = len(spans)
r = body(cur, I)
if not r[0]:
if len(spans) != mark_i:
del spans[mark_i:]
break
if body_raw:
parts.append(flatten(r[1], False))
else:
parts.append(r[1])
cur = r[2]
count += 1
if count < mn:
if cur > I.farthest:
I.farthest = cur
if len(spans) != mark:
del spans[mark:]
return _FAIL
if maybe:
first = parts[0] if parts else None
if named or not (first is None or first == ""):
return (True, first, cur)
return (True, "", cur)
return (True, _flatten_repetition(parts, named), cur)

return rep

def c_cap(body, name, is_span_cap, body_raw):
# body_raw: the body's value is a tagged list (a raw-marked rule)
# that CanFlatten must resolve, with the named flag, before the
Expand Down Expand Up @@ -1239,6 +1310,10 @@ def build(node, raw=False, named=False):
ranges = _merge_ranges([r for ch in branches for r in ch.ranges])
return c_cc(ranges, _charset_for(ranges))
children = tuple(build(ch, raw, named) for ch in branches)
# WO19: a spanning alternation whose branch fails after a
# later branch's span caps recorded must roll them back.
if node_spanning(node):
return c_alt_span(children)
return c_alt(children)
if t is Seq:
if raw:
Expand All @@ -1248,6 +1323,8 @@ def build(node, raw=False, named=False):
(build(ch, False, False), type(ch) is Ref and ch.name in raw_rules)
for ch in node.a
)
if node_spanning(node):
return c_seq_final_span(children)
return c_seq_final(children)
if t is Rep:
body = node.a
Expand All @@ -1261,6 +1338,7 @@ def build(node, raw=False, named=False):
and node.min == 0
and node.max == 1
and type(body) is Ref
and not node_spanning(node)
):
wrap = maybe_wrap_of(body.name)
if wrap is not None:
Expand Down Expand Up @@ -1295,6 +1373,10 @@ def build(node, raw=False, named=False):
if raw:
return c_rep(build(body, True, False), node.min, node.max, node.maybe)
body_raw = type(body) is Ref and body.name in raw_rules
if node_spanning(node):
return c_rep_final_span(
build(body, False, False), body_raw, node.min, node.max, node.maybe, named
)
return c_rep_final(
build(body, False, False), body_raw, node.min, node.max, node.maybe, named
)
Expand Down
14 changes: 11 additions & 3 deletions interscript-py/src/interscript/isc.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,21 @@
Parser fidelity notes:

- The generated parser reproduces the former hand parser's documents
byte-for-byte across the whole maps corpus (434/434 through the
byte-for-byte across the whole maps corpus (432/435 through the
py-vs-py parity driver, scripts/parse_parity_py.py), including the
§4.2.6 trailing-parallel override join and byte-exact `from_source`
override keys. Documented hand-parser quirks the grammar rejects
(invalid `\\u` escapes, hyphenated identifiers outside run targets,
multi-digit capture refs, stage-level reverse/strip/swapcase) are
corpus-dormant; see grammar/actions.py for the full deviation list.
multi-digit capture refs) are corpus-dormant; see grammar/actions.py
for the full deviation list. (WO19 G6: stage-level
reverse/strip/swapcase moved from rejected quirk to IMPLEMENTED
directive; the capital `\\UXXXX` escape now decodes, and the
outside-form block-rule `when` extension was retired — canonical rs
binds the guard inside the braces, ruby parses the inside form only.)
The three py-vs-py holdouts (gki-bel, mvd-bel, odni-bel) are the
capital-`\\U` maps: the generated parser decodes the escape (ruby/rs
canon) where the hand parser carried the literal digits — the hand
side is the bug WO19 G6 item 4 fixes.
- `languages` is declared once per map; a second declaration is a parse
error. `use` may appear in stage bodies, ruleset bodies and inside
`parallel`/`sequence` blocks (WO18 G5, spec §3.2): inside a pass block
Expand Down
18 changes: 18 additions & 0 deletions interscript-py/src/interscript/items.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@
from .errors import MapLogicError
from .stdlib import regexp_escape

# WO19 G6: the six fn literals are never from/constraint values — they
# execute only as stage directives or `to` funcalls (FuncallInline).
# ruby raises (canon), rs raises at build; the former alias fallback
# degraded to a misleading "Alias <name> not found". `any_character` is
# NOT in this set: it is the primitive spelled as a function atom
# (rs lowers it to the any_character stdlib alias).
FN_LITERALS = frozenset(
{"upcase", "downcase", "title_case", "reverse", "strip", "swapcase"}
)


@dataclass(frozen=True)
class CompiledItem:
Expand Down Expand Up @@ -96,6 +106,14 @@ def compile_item(
if item.name in ALIAS_COMPILED:
a = _alias_compiled(item.name)
return _compiled(a.re, a.literal)
# WO19 G6 item 2: a fn literal in FROM position is map-logic
# damage — raise the named error instead of the misleading
# "Alias <name> not found".
if item.name in FN_LITERALS:
raise MapLogicError(
f'function literal "{item.name}" in from position '
"(fn literals are directives or to-funcalls, not from values)"
)
resolved = ctx.resolve_alias(item.name)
# WO18 G3 (parity audit): an unresolved alias is map-logic
# damage, not an empty matcher. Compiling re="" matched at every
Expand Down
27 changes: 27 additions & 0 deletions interscript-py/src/interscript/stdlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,27 @@ def decompose(text: str, opts: dict | None = None) -> str:
return unicodedata.normalize("NFD", text)


# Ruby String#strip's character set: null, horizontal tab, line feed,
# vertical tab, form feed, carriage return, space (ASCII set only —
# deliberately NOT python's default Unicode-whitespace strip).
_RUBY_STRIP_SET = "\0\t\n\v\f\r "


def reverse(text: str, opts: dict | None = None) -> str:
"""Ruby Stdlib::Functions.reverse = String#reverse (WO19 G6)."""
return text[::-1]


def strip(text: str, opts: dict | None = None) -> str:
"""Ruby Stdlib::Functions.strip = String#strip (WO19 G6)."""
return text.strip(_RUBY_STRIP_SET)


def swapcase(text: str, opts: dict | None = None) -> str:
"""Ruby Stdlib::Functions.swapcase = String#swapcase (WO19 G6)."""
return text.swapcase()


class Function(Protocol):
def __call__(self, text: str, opts: dict | None = None) -> str: ...

Expand All @@ -202,6 +223,12 @@ def __call__(self, text: str, opts: dict | None = None) -> str: ...
"downcase": downcase,
"upcase": upcase,
"title_case": title_case,
# WO19 G6: the three new fn literals execute as directives and
# inline to-funcalls with the plain String semantics of ruby
# Stdlib::Functions.
"reverse": reverse,
"strip": strip,
"swapcase": swapcase,
"separate": separate,
"compose": compose,
"decompose": decompose,
Expand Down
43 changes: 43 additions & 0 deletions interscript-py/tests/test_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
Stage,
StringItem,
SubRule,
FuncallInline,
)
from interscript.errors import AsyncFunctionError, MapLogicError
from interscript.executor import execute_stage
Expand Down Expand Up @@ -281,3 +282,45 @@ def test_repeat_maybe(self):
)
m = make_map([stage("main", [rule])])
assert execute_stage(m, "main", "a ab abb") == "X X Xb"


class TestWO19G6FnLiterals:
"""WO19 G6 — fn-literal alignment (items 1-3)."""

def test_fn_literal_in_from_raises_named_error(self):
# Item 2: a fn literal in FROM position is map-logic damage —
# ruby raises (canon), rs raises at build; the former alias
# fallback said "Alias downcase not found".
m = make_map([stage("main", [sub(AliasItem(name="downcase"), "X")])])
with pytest.raises(
MapLogicError,
match='function literal "downcase" in from position',
):
execute_stage(m, "main", "abc")

def test_reverse_directive_executes(self):
# Item 3: ruby String#reverse semantics.
m = make_map(
[stage("main", [sub("1", "2"), FuncallRule(name="reverse")])]
)
assert execute_stage(m, "main", "abc1") == "2cba"

def test_strip_directive_executes(self):
# Item 3: ruby String#strip semantics (ASCII whitespace + NUL).
m = make_map([stage("main", [FuncallRule(name="strip")])])
assert execute_stage(m, "main", " x ") == "x"
assert execute_stage(m, "main", "\t y \n") == "y"

def test_swapcase_directive_executes(self):
# Item 3: ruby String#swapcase semantics.
m = make_map(
[stage("main", [sub("a", "b"), FuncallRule(name="swapcase")])]
)
assert execute_stage(m, "main", "aBc") == "BbC"

def test_reverse_inline_to_funcall_executes(self):
# Item 3: a reverse fn literal as a `to` target.
m = make_map(
[stage("main", [SubRule(from_=StringItem(value="abc"), to=FuncallInline(name="reverse"))])]
)
assert execute_stage(m, "main", "xabcy") == "xcbay"
Loading
Loading