From 395d1555d9c4f992f1f3892ecbccca568c5d2989 Mon Sep 17 00:00:00 2001 From: Jon Ruttan Date: Fri, 6 Nov 2026 10:50:51 -0500 Subject: [PATCH] perf(lexer): lowercase string prefixes only, so a cold first prompt fits The painter's Lexer is made at a terminal session's first prompt. With an empty code cache that make compiles every state and then records the plan for its cache group, and the plan's lookups are linear walks done once a state, so its cost grows with the square of the states. The 25 prefixes of both cases gave 100 string rules and 686 states: the plan alone allocated 43M objects and the first prompt peaked at 2,986 MB (macOS footprint, x-lang v0.47.0), which is what x-os's warm step was killed for in its 3.6 GB podman VM. The 9 lowercase prefixes give 254 states and a cold first prompt of 745 MB. An uppercase prefix now paints as a name, then its string. Co-Authored-By: Claude Opus 5.5 --- python/lexer.x | 13 +++++++------ tests/specs/09-line.spec.md | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/lexer.x b/python/lexer.x index 2428b87..a785c17 100644 --- a/python/lexer.x +++ b/python/lexer.x @@ -17,9 +17,12 @@ ; whose body takes a backslash escape, a triple-quoted one an until span, ; which ends at its first whole `"""` or `'''`. Both run to the end of the ; text while unclosed, and so does a comment. A Lexer literal is bytes, so -; each prefix Python takes -- r, b, u, f, rb, br, fr and rf, in either case -; -- opens with a literal of its own, and a prefix with no quote after it is -; a name. +; each prefix -- r, b, u, f, rb, br, fr and rf -- opens with a literal of its +; own, and a prefix with no quote after it is a name. Lowercase only: every +; literal adds string states, and the plan the Lexer records for its cache +; group is quadratic in the states. The 25 prefixes of both cases made 686 +; states and a 3.0 GB first prompt on an empty code cache; these 9 make 254 +; and 0.73 GB. An uppercase prefix paints as a name, then its string. ; ; The tags: blank, comment, string, number, name, decorator and plain. The ; module imports x-lang libraries only, so a process colouring a file loads @@ -39,9 +42,7 @@ (def %py-lex-num-mantissa (list 46 95 (pair 48 57))) (def %py-lex-prefixes - (list "" "r" "R" "b" "B" "u" "U" "f" "F" - "rb" "rB" "Rb" "RB" "br" "bR" "Br" "BR" - "fr" "fR" "Fr" "FR" "rf" "rF" "Rf" "RF")) + (list "" "r" "b" "u" "f" "rb" "br" "fr" "rf")) (def %py-lex-bref (prim-ref (lit str) (lit byte-ref))) (def %py-lex-cint (prim-ref (lit char) (lit ->int))) diff --git a/tests/specs/09-line.spec.md b/tests/specs/09-line.spec.md index 6f25fb5..4702e66 100644 --- a/tests/specs/09-line.spec.md +++ b/tests/specs/09-line.spec.md @@ -92,7 +92,7 @@ a string green, ``; a comment dim, ``. "t = \"\"\"open ' \" triple")) ``` --- - ("r'\\d+' + b'x' + u\"y\" + F\"z\" + BR\"w\" + bu'no'" "\"a\\\"b\" + 'c\\'d'" "x = \"\"\"abc\"\"\" + 1" "x = '''a'b''' + 2" "s = 'abc" "x = \"unterminated # not a comment" "t = \"\"\"open ' \" triple") + ("r'\\d+' + b'x' + u\"y\" + F\"z\" + BR\"w\" + bu'no'" "\"a\\\"b\" + 'c\\'d'" "x = \"\"\"abc\"\"\" + 1" "x = '''a'b''' + 2" "s = 'abc" "x = \"unterminated # not a comment" "t = \"\"\"open ' \" triple") ### a comment runs to the end, a decorator is one span, and operators stay plain