diff --git a/lang.xon b/lang.xon index c756a99..6cf2b04 100644 --- a/lang.xon +++ b/lang.xon @@ -11,6 +11,7 @@ ; (dialect NAME) he | xe | rn -- a REQUIREMENT, not a taste ; (requires-release "TAG") the x-lang this bundle was built against ; (entry "FILE") loaded AFTER the dialect, not instead of it +; (grammar NAME (module M) ...) a highlighting grammar x/tool/syntax loads (lang "python") @@ -63,3 +64,7 @@ (requires-release "v0.46.0") (entry "run.x") + +; The Python grammar, for any process that colours a file: python/syntax +; imports x-lang libraries and python/lexer only, never the interpreter. +(grammar python (module python/syntax) (files ".py" ".pyw" ".pyi") (interpreters "python" "python3")) diff --git a/python/lexer.x b/python/lexer.x new file mode 100644 index 0000000..2428b87 --- /dev/null +++ b/python/lexer.x @@ -0,0 +1,87 @@ +; # x-python -- Python on x-lang +; +; ## python/lexer.x -- the Lexer rules that colour Python +; +; @description One set of Lexer rules for colouring Python, read by the +; line editor's painter and by the highlighting grammar alike. +; @author [Jon Ruttan](jonruttan@gmail.com) +; @copyright 2026 Jon Ruttan +; @license MIT No Attribution (MIT-0) +; +; Python's tokenizer is a second base: it answers tokens, not spans, and it +; stops at an unterminated string, which is the normal state of a line being +; typed or of a window onto a file. These rules read every byte instead. +; Blanks are a run, and a bracket or an operator is one byte under `plain`, +; so a painter draws a line byte for byte with the author's spacing. A +; string is one token with its prefix: a single-quoted one a nested span +; whose body takes a backslash escape, a triple-quoted one an until span, +; which ends at its first whole `"""` or `'''`. Both run to the end of the +; text while unclosed, and so does a comment. A Lexer literal is bytes, so +; each prefix Python takes -- r, b, u, f, rb, br, fr and rf, in either case +; -- opens with a literal of its own, and a prefix with no quote after it is +; a name. +; +; The tags: blank, comment, string, number, name, decorator and plain. The +; module imports x-lang libraries only, so a process colouring a file loads +; it without the interpreter. +(module python/lexer) +(import x/type/list) +(import x/type/str) +(import x/reader/lexer) + +(def %py-lex-ident-start (list 95 (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1))) +(def %py-lex-ident (list 95 (pair 48 57) (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1))) +(def %py-lex-digits (list (pair 48 57))) +; a number's body: digits, letters, underscores and dots, so 0x1f, 1_000, +; 1e5j and 1.5 are each one token +(def %py-lex-num-body (list 46 95 (pair 48 57) (pair 65 90) (pair 97 122))) +; the body before an exponent letter, so the sign after it is read as well +(def %py-lex-num-mantissa (list 46 95 (pair 48 57))) + +(def %py-lex-prefixes + (list "" "r" "R" "b" "B" "u" "U" "f" "F" + "rb" "rB" "Rb" "RB" "br" "bR" "Br" "BR" + "fr" "fR" "Fr" "FR" "rf" "rF" "Rf" "RF")) + +(def %py-lex-bref (prim-ref (lit str) (lit byte-ref))) +(def %py-lex-cint (prim-ref (lit char) (lit ->int))) + +(def %py-lex-string-rules + (fn (_) + (List flat-map + (fn (_ p) + (List flat-map + (fn (_ q) + (let ((q3 (Str8 append q q q))) + (list (Lexer until (lit string) (Str8 append p q3) q3 (lit to-end)) + (Lexer nested (lit string) (Str8 append p q) (lit body) + (list (list (lit body) (%py-lex-cint (%py-lex-bref q 0)) 92 ())) (lit to-end))))) + (list "\"" "'"))) + %py-lex-prefixes))) + +; A number is digits, or a dot before a digit, then its body; the second +; pattern of each pair reads an exponent's sign, which the body alone would +; leave, and wins by length when there is one. +(def python-lexer-rules + (fn (_) + (List append + (list (Lexer run (lit blank) " \t\r\n" " \t\r\n") + (Lexer until (lit comment) "#" "\n" (lit to-end))) + (%py-lex-string-rules) + (list + (Lexer pattern (lit number) + (list (list %py-lex-digits 1 1) (list %py-lex-num-body 0 ()))) + (Lexer pattern (lit number) + (list (list %py-lex-digits 1 1) (list %py-lex-num-mantissa 0 ()) (list "eE" 1 1) + (list "+-" 1 1) (list %py-lex-num-body 0 ()))) + (Lexer pattern (lit number) + (list (list "." 1 1) (list %py-lex-digits 1 1) (list %py-lex-num-body 0 ()))) + (Lexer pattern (lit number) + (list (list "." 1 1) (list %py-lex-digits 1 1) (list %py-lex-num-mantissa 0 ()) (list "eE" 1 1) + (list "+-" 1 1) (list %py-lex-num-body 0 ()))) + (Lexer run (lit name) %py-lex-ident-start %py-lex-ident) + (Lexer pattern (lit decorator) + (list (list "@" 1 1) (list %py-lex-ident-start 1 1) (list %py-lex-ident 0 ()))) + (Lexer any (lit plain)))))) + +(provide python/lexer python-lexer-rules) diff --git a/python/line.x b/python/line.x index ff4ef8d..d3fa344 100644 --- a/python/line.x +++ b/python/line.x @@ -24,7 +24,7 @@ ; Python's tokenizer is a second base: it answers tokens, not spans, and it ; stops at an unterminated string, which is the normal state of a line ; being typed. So this file colours a line by the tokens of a Lexer -; (x/reader/lexer) of its own whose rules read every byte, through the +; (x/reader/lexer) made from python/lexer's rules, which read every byte, through the ; platform's (Paint lexer): comments, strings with their prefixes and triple ; quotes, numbers, names, decorators, and every other byte as itself. The ; bracket marks are read from the same tokens. A name is classified @@ -53,6 +53,7 @@ (import x/type/str) (import x/type/list) (import x/reader/lexer) +(import python/lexer python-lexer-rules) (import x/repl/ansi) (import x/repl/paint) ; The editor. Imported here rather than left to the launcher so that its @@ -150,72 +151,11 @@ ; --- the lexer --------------------------------------------------------------- ; -; Every byte of a line is some token's: blanks a run, a bracket or an -; operator one byte under `plain`, so the painter draws the line byte for -; byte with the author's spacing. A string is one token with its prefix: a -; single-quoted one a nested span whose body takes a backslash escape, a -; triple-quoted one an until span, which ends at its first whole `"""` or -; `'''`. Both run to the end of the text while unclosed, the normal state -; of a line being typed, and so does a comment. A Lexer literal is bytes, -; so each prefix Python takes -- r, b, u, f, rb, br, fr and rf, in either -; case -- opens with a literal of its own, and a prefix with no quote after -; it is a name. -(def %pp-ident-start (list 95 (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1))) -(def %pp-ident (list 95 (pair 48 57) (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1))) -(def %pp-digits (list (pair 48 57))) -; a number's body: digits, letters, underscores and dots, so 0x1f, 1_000, -; 1e5j and 1.5 are each one token -(def %pp-num-body (list 46 95 (pair 48 57) (pair 65 90) (pair 97 122))) -; the body before an exponent letter, so the sign after it is read as well -(def %pp-num-mantissa (list 46 95 (pair 48 57))) - -(def %pp-prefixes - (list "" "r" "R" "b" "B" "u" "U" "f" "F" - "rb" "rB" "Rb" "RB" "br" "bR" "Br" "BR" - "fr" "fR" "Fr" "FR" "rf" "rF" "Rf" "RF")) - -(def %pp-string-rules - (fn (_) - (List flat-map - (fn (_ p) - (List flat-map - (fn (_ q) - (let ((q3 (Str8 append q q q))) - (list (Lexer until (lit string) (Str8 append p q3) q3 (lit to-end)) - (Lexer nested (lit string) (Str8 append p q) (lit body) - (list (list (lit body) (%pp-byte q 0) 92 ())) (lit to-end))))) - (list "\"" "'"))) - %pp-prefixes))) - -; A number is digits, or a dot before a digit, then its body; the second -; pattern of each pair reads an exponent's sign, which the body alone would -; leave, and wins by length when there is one. -(def %pp-rules - (fn (_) - (List append - (list (Lexer run (lit blank) " \t\r\n" " \t\r\n") - (Lexer until (lit comment) "#" "\n" (lit to-end))) - (%pp-string-rules) - (list - (Lexer pattern (lit number) - (list (list %pp-digits 1 1) (list %pp-num-body 0 ()))) - (Lexer pattern (lit number) - (list (list %pp-digits 1 1) (list %pp-num-mantissa 0 ()) (list "eE" 1 1) - (list "+-" 1 1) (list %pp-num-body 0 ()))) - (Lexer pattern (lit number) - (list (list "." 1 1) (list %pp-digits 1 1) (list %pp-num-body 0 ()))) - (Lexer pattern (lit number) - (list (list "." 1 1) (list %pp-digits 1 1) (list %pp-num-mantissa 0 ()) (list "eE" 1 1) - (list "+-" 1 1) (list %pp-num-body 0 ()))) - (Lexer run (lit name) %pp-ident-start %pp-ident) - (Lexer pattern (lit decorator) - (list (list "@" 1 1) (list %pp-ident-start 1 1) (list %pp-ident 0 ()))) - (Lexer any (lit plain)))))) - +; python/lexer's rules, the ones the highlighting grammar reads too. (def %pp-lexer ()) (def %pp-lex (fn (_) - (if (null? %pp-lexer) (set! %pp-lexer (Lexer make (%pp-rules))) ()) + (if (null? %pp-lexer) (set! %pp-lexer (Lexer make (python-lexer-rules))) ()) %pp-lexer)) ; Each tag's colour: the code strings above, read when a token is painted. diff --git a/python/syntax.x b/python/syntax.x new file mode 100644 index 0000000..35e702b --- /dev/null +++ b/python/syntax.x @@ -0,0 +1,115 @@ +; # x-python -- Python on x-lang +; +; ## python/syntax.x -- the Python highlighting grammar +; +; @description The grammar x/tool/syntax colours Python files with: the line +; editor's Lexer rules, with names classed as keywords, constants, +; builtins and classes. +; @author [Jon Ruttan](jonruttan@gmail.com) +; @copyright 2026 Jon Ruttan +; @license MIT No Attribution (MIT-0) +; +; lang.xon's (grammar python ...) form names this module, and x/tool/syntax +; imports it the first time a file needs the grammar, in whatever process is +; colouring: vi's, a REPL's, an HTML renderer's. So it imports x-lang +; libraries and python/lexer, never the interpreter, and the words it colours +; are its own copy of the interpreter's tables. A spec holds the two to the +; same words. +; +; A name is classed as the line editor's painter classes it: a keyword, then +; True, False and None as constants, then a builtin, then a leading capital +; as a class. A decorator is a keyword. A bracket, a comma, a colon, a +; semicolon or a dot is punctuation, every other plain byte an operator. + +(module python/syntax) +(import x/type/list) +(import x/type/str) +(import x/tool/syntax) +(import python/lexer python-lexer-rules) + +; The keywords python/tokens.x reads, as python-keywords lists them. +(def python-syntax-keywords + (list "if" "elif" "else" "while" "def" "return" "pass" "and" "or" "not" + "in" "is" "for" "break" "continue" "class" "import" "from" "as" + "try" "except" "finally" "raise" "with" "lambda" "global" "nonlocal" + "assert" "del" "yield" "async" "await")) + +(def python-syntax-constants (list "True" "False" "None")) + +; The builtins python/parse.x's python-builtins table names, each once, in +; the table's order, less the constants. +(def python-syntax-builtins + (list "print" "__debug__" "len" "range" "slice" "str" "repr" "list" "hasattr" + "object" "bytes" "staticmethod" "classmethod" "property" "type" "int" + "float" "bool" "dict" "tuple" "isinstance" "pow" "abs" "round" "min" + "max" "bytearray" "memoryview" "complex" "hash" "NotImplemented" + "Ellipsis" "eval" "exec" "compile" "dir" "globals" "locals" "vars" "chr" + "ord" "StopIteration" "StopAsyncIteration" "GeneratorExit" "next" "sum" + "map" "zip" "all" "any" "sorted" "iter" "set" "frozenset" "bin" "hex" + "oct" "divmod" "callable" "__import__" "__build_class__" "id" "getattr" + "setattr" "delattr" "issubclass" "enumerate" "filter" "reversed" + "SystemExit" "BaseException" "Exception" "ImportError" + "ModuleNotFoundError" "OSError" "FileNotFoundError" "PermissionError" + "FileExistsError" "IsADirectoryError" "open" "IOError" "EOFError" + "KeyboardInterrupt" "IndentationError" "UnicodeError" + "UnicodeDecodeError" "UnicodeEncodeError" "MemoryError" "OverflowError" + "NotImplementedError" "ArithmeticError" "LookupError" + "ZeroDivisionError" "IndexError" "KeyError" "AttributeError" "NameError" + "UnboundLocalError" "TypeError" "ValueError" "AssertionError" + "RuntimeError" "SyntaxError")) + +(def %py-syntax-bref (prim-ref (lit str) (lit byte-ref))) +(def %py-syntax-cint (prim-ref (lit char) (lit ->int))) +(def %py-syntax-blen (prim-ref (lit str) (lit byte-len))) +(def %py-syntax-byte (fn (_ s i) (%py-syntax-cint (%py-syntax-bref s i)))) +(def %py-syntax-upper? (fn (_ b) (if (>= b 65) (<= b 90) #f))) +(def %py-syntax-lower? (fn (_ b) (if (>= b 97) (<= b 122) #f))) + +(def %py-syntax-name-class + (fn (_ text) + (match + ((List includes? text python-syntax-keywords) (lit keyword)) + ((List includes? text python-syntax-constants) (lit constant)) + ((List includes? text python-syntax-builtins) (lit name)) + ((%py-syntax-upper? (%py-syntax-byte text 0)) (lit class)) + (#t ())))) + +; ( ) , . : ; [ ] { } +(def %py-syntax-punct (list 40 41 44 46 58 59 91 93 123 125)) + +; A blank is plain only at the text's end, where the blank run meets the +; Lexer's end text and is not a token; it has no class. +(def %py-syntax-plain-class + (fn (_ text) + (let ((b (%py-syntax-byte text 0))) + (match + ((<= b 32) ()) + ((List includes? b %py-syntax-punct) (lit punct)) + (#t (lit operator)))))) + +; A line starting with a name, a comment or a decorator at its first column +; is taken to start outside any string, so spans need not read the file from +; its top. A triple-quoted string with such a line in it colours inverted +; from that line to the end of the window, as other editors' Python grammars +; do. +(def %py-syntax-sync + (fn (_ s ls) + (if (>= ls (%py-syntax-blen s)) #f + (let ((b (%py-syntax-byte s ls))) + (match + ((List includes? b (list 35 64 95)) #t) + ((%py-syntax-upper? b) #t) + (#t (%py-syntax-lower? b))))))) + +(Syntax register! + (let ((g (Syntax make (lit python) (python-lexer-rules) + (list (pair (lit comment) (lit comment)) + (pair (lit string) (lit string)) + (pair (lit number) (lit number)) + (pair (lit decorator) (lit keyword)) + (pair (lit name) %py-syntax-name-class) + (pair (lit plain) %py-syntax-plain-class))))) + (g sync %py-syntax-sync) + g)) + +(provide python/syntax python-syntax-keywords python-syntax-constants python-syntax-builtins) diff --git a/tests/specs/C0-syntax.spec.md b/tests/specs/C0-syntax.spec.md new file mode 100644 index 0000000..ca58649 --- /dev/null +++ b/tests/specs/C0-syntax.spec.md @@ -0,0 +1,98 @@ +## python/syntax -- the Python highlighting grammar + +The bundle's lang.xon declares a grammar, `python`, that x/tool/syntax +colours Python files with. Its module, `python/syntax`, imports x-lang +libraries and python/lexer only, and registers the grammar as it loads. + +The cases share one process, and a registered grammar stays registered, so +the catalogue's case comes first: it finds the grammar through lang.xon, +before anything has imported the module. + +### the catalogue finds python by suffix, by #! and by modeline + +The langs directory holds this bundle, linked as `python`. The row comes from +lang.xon, its module not loaded until a file asks for the grammar, and the +grammar takes lang.xon's suffixes and interpreters. + +```python +(do (import x/sys/posix) (import x/sys/file) (import x/tool/syntax) + (def root (Str8 replace "/python/syntax.x" "" (%module-resolve-file "python/syntax.x"))) + (def tmp (File temp "/tmp/x-python-spec-langs-")) + (File close (first tmp)) + (def d (rest tmp)) + (File unlink d) + (File mkdir d) + (File symlink root (Str8 append d "/python")) + (def was (Sys getenv "X_LANG_DIR")) + (Sys setenv "X_LANG_DIR" d) + (Syntax forget!) + (def row (List find (fn (_ r) (eq? (first r) (lit python))) (Syntax catalogue))) + (def name (fn (_ path . text) + (let ((g (apply Syntax (pair (lit for-file) (pair path text))))) (if (null? g) () (g name))))) + (def seen + (list (List ref 1 row) + (name "x.py") (name "stubs/x.pyi") + (name "run" "#!/usr/bin/env python3\nprint(1)\n") + (name "run" "x = 1\n# vi: set ft=python:\n") + (name "x.txt") + (let ((g (Syntax named (lit python)))) (list (g files) (g interpreters))))) + (File unlink (Str8 append d "/python")) + (File rmdir d) + (Sys setenv "X_LANG_DIR" (if (null? was) "" was)) + (Syntax forget!) + seen) +``` +--- + ('python/syntax 'python 'python 'python 'python () ((".py" ".pyw" ".pyi") ("python" "python3"))) + +### a file's spans: comments, decorators, strings, numbers and names + +A keyword, True, False and None, a builtin and a capitalised name each have a +class; any other name has none. Brackets, commas, colons and dots are +punctuation, the other plain bytes operators. + +```python +(do (import x/tool/syntax) (import python/syntax) + ((Syntax named (lit python)) spans + "# c\n@dec\ndef f(x=1.5e-3):\n return len(x), None, Foo, r'a\\'b' + y\n")) +``` +--- + ((0 3 'comment) (4 4 'keyword) (9 3 'keyword) (14 1 'punct) (16 1 'operator) (17 6 'number) (23 1 'punct) (24 1 'punct) (30 6 'keyword) (37 3 'name) (40 1 'punct) (42 1 'punct) (43 1 'punct) (45 4 'constant) (49 1 'punct) (51 3 'class) (54 1 'punct) (56 7 'string) (64 1 'operator)) + +### strings with prefixes, triple quotes, and one left open + +```python +(do (import x/tool/syntax) (import python/syntax) + (List map (fn (_ s) ((Syntax named (lit python)) spans s)) + (list "b'\\x00' f\"{a}\" rb'''x'''" "s = \"\"\"doc\nmore" "t = 'open"))) +``` +--- + (((0 7 'string) (8 6 'string) (15 9 'string)) ((2 1 'operator) (4 11 'string)) ((2 1 'operator) (4 5 'string))) + +### the grammar's words are the interpreter's keywords and builtins + +```python +(do (import python/syntax python-syntax-keywords python-syntax-constants python-syntax-builtins) + (import python/tokens python-keywords) + (import python/core python-builtins) + (def table (List map first python-builtins)) + (def ours (List append python-syntax-constants python-syntax-builtins)) + (list (equal? python-syntax-keywords python-keywords) + (List filter (fn (_ n) (not (List includes? n ours))) table) + (List filter (fn (_ n) (not (List includes? n table))) ours))) +``` +--- + (#t () ()) + +### a window reads from the line above it at the first column + +Spans walk back from a window's first line to one starting with a name, a +comment or a decorator, so a triple-quoted string opened on an indented line +above colours the window's start. + +```python +(do (import x/tool/syntax) (import python/syntax) + ((Syntax named (lit python)) spans "def f():\n s = '''a\n b''' + 1\n" 23 9)) +``` +--- + ((23 7 'string) (31 1 'operator))