Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions lang.xon
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
; (dialect NAME) he | xe | rn -- a REQUIREMENT, not a taste
; (requires-release "TAG") the x-lang this bundle was built against
; (entry "FILE") loaded AFTER the dialect, not instead of it
; (grammar NAME (module M) ...) a highlighting grammar x/tool/syntax loads

(lang "python")

Expand Down Expand Up @@ -63,3 +64,7 @@
(requires-release "v0.46.0")

(entry "run.x")

; The Python grammar, for any process that colours a file: python/syntax
; imports x-lang libraries and python/lexer only, never the interpreter.
(grammar python (module python/syntax) (files ".py" ".pyw" ".pyi") (interpreters "python" "python3"))
87 changes: 87 additions & 0 deletions python/lexer.x
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
; # x-python -- Python on x-lang
;
; ## python/lexer.x -- the Lexer rules that colour Python
;
; @description One set of Lexer rules for colouring Python, read by the
; line editor's painter and by the highlighting grammar alike.
; @author [Jon Ruttan](jonruttan@gmail.com)
; @copyright 2026 Jon Ruttan
; @license MIT No Attribution (MIT-0)
;
; Python's tokenizer is a second base: it answers tokens, not spans, and it
; stops at an unterminated string, which is the normal state of a line being
; typed or of a window onto a file. These rules read every byte instead.
; Blanks are a run, and a bracket or an operator is one byte under `plain`,
; so a painter draws a line byte for byte with the author's spacing. A
; string is one token with its prefix: a single-quoted one a nested span
; whose body takes a backslash escape, a triple-quoted one an until span,
; which ends at its first whole `"""` or `'''`. Both run to the end of the
; text while unclosed, and so does a comment. A Lexer literal is bytes, so
; each prefix Python takes -- r, b, u, f, rb, br, fr and rf, in either case
; -- opens with a literal of its own, and a prefix with no quote after it is
; a name.
;
; The tags: blank, comment, string, number, name, decorator and plain. The
; module imports x-lang libraries only, so a process colouring a file loads
; it without the interpreter.
(module python/lexer)
(import x/type/list)
(import x/type/str)
(import x/reader/lexer)

(def %py-lex-ident-start (list 95 (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1)))
(def %py-lex-ident (list 95 (pair 48 57) (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1)))
(def %py-lex-digits (list (pair 48 57)))
; a number's body: digits, letters, underscores and dots, so 0x1f, 1_000,
; 1e5j and 1.5 are each one token
(def %py-lex-num-body (list 46 95 (pair 48 57) (pair 65 90) (pair 97 122)))
; the body before an exponent letter, so the sign after it is read as well
(def %py-lex-num-mantissa (list 46 95 (pair 48 57)))

(def %py-lex-prefixes
(list "" "r" "R" "b" "B" "u" "U" "f" "F"
"rb" "rB" "Rb" "RB" "br" "bR" "Br" "BR"
"fr" "fR" "Fr" "FR" "rf" "rF" "Rf" "RF"))

(def %py-lex-bref (prim-ref (lit str) (lit byte-ref)))
(def %py-lex-cint (prim-ref (lit char) (lit ->int)))

(def %py-lex-string-rules
(fn (_)
(List flat-map
(fn (_ p)
(List flat-map
(fn (_ q)
(let ((q3 (Str8 append q q q)))
(list (Lexer until (lit string) (Str8 append p q3) q3 (lit to-end))
(Lexer nested (lit string) (Str8 append p q) (lit body)
(list (list (lit body) (%py-lex-cint (%py-lex-bref q 0)) 92 ())) (lit to-end)))))
(list "\"" "'")))
%py-lex-prefixes)))

; A number is digits, or a dot before a digit, then its body; the second
; pattern of each pair reads an exponent's sign, which the body alone would
; leave, and wins by length when there is one.
(def python-lexer-rules
(fn (_)
(List append
(list (Lexer run (lit blank) " \t\r\n" " \t\r\n")
(Lexer until (lit comment) "#" "\n" (lit to-end)))
(%py-lex-string-rules)
(list
(Lexer pattern (lit number)
(list (list %py-lex-digits 1 1) (list %py-lex-num-body 0 ())))
(Lexer pattern (lit number)
(list (list %py-lex-digits 1 1) (list %py-lex-num-mantissa 0 ()) (list "eE" 1 1)
(list "+-" 1 1) (list %py-lex-num-body 0 ())))
(Lexer pattern (lit number)
(list (list "." 1 1) (list %py-lex-digits 1 1) (list %py-lex-num-body 0 ())))
(Lexer pattern (lit number)
(list (list "." 1 1) (list %py-lex-digits 1 1) (list %py-lex-num-mantissa 0 ()) (list "eE" 1 1)
(list "+-" 1 1) (list %py-lex-num-body 0 ())))
(Lexer run (lit name) %py-lex-ident-start %py-lex-ident)
(Lexer pattern (lit decorator)
(list (list "@" 1 1) (list %py-lex-ident-start 1 1) (list %py-lex-ident 0 ())))
(Lexer any (lit plain))))))

(provide python/lexer python-lexer-rules)
68 changes: 4 additions & 64 deletions python/line.x
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
; Python's tokenizer is a second base: it answers tokens, not spans, and it
; stops at an unterminated string, which is the normal state of a line
; being typed. So this file colours a line by the tokens of a Lexer
; (x/reader/lexer) of its own whose rules read every byte, through the
; (x/reader/lexer) made from python/lexer's rules, which read every byte, through the
; platform's (Paint lexer): comments, strings with their prefixes and triple
; quotes, numbers, names, decorators, and every other byte as itself. The
; bracket marks are read from the same tokens. A name is classified
Expand Down Expand Up @@ -53,6 +53,7 @@
(import x/type/str)
(import x/type/list)
(import x/reader/lexer)
(import python/lexer python-lexer-rules)
(import x/repl/ansi)
(import x/repl/paint)
; The editor. Imported here rather than left to the launcher so that its
Expand Down Expand Up @@ -150,72 +151,11 @@

; --- the lexer ---------------------------------------------------------------
;
; Every byte of a line is some token's: blanks a run, a bracket or an
; operator one byte under `plain`, so the painter draws the line byte for
; byte with the author's spacing. A string is one token with its prefix: a
; single-quoted one a nested span whose body takes a backslash escape, a
; triple-quoted one an until span, which ends at its first whole `"""` or
; `'''`. Both run to the end of the text while unclosed, the normal state
; of a line being typed, and so does a comment. A Lexer literal is bytes,
; so each prefix Python takes -- r, b, u, f, rb, br, fr and rf, in either
; case -- opens with a literal of its own, and a prefix with no quote after
; it is a name.
(def %pp-ident-start (list 95 (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1)))
(def %pp-ident (list 95 (pair 48 57) (pair 65 90) (pair 97 122) (pair 128 255) (pair -128 -1)))
(def %pp-digits (list (pair 48 57)))
; a number's body: digits, letters, underscores and dots, so 0x1f, 1_000,
; 1e5j and 1.5 are each one token
(def %pp-num-body (list 46 95 (pair 48 57) (pair 65 90) (pair 97 122)))
; the body before an exponent letter, so the sign after it is read as well
(def %pp-num-mantissa (list 46 95 (pair 48 57)))

(def %pp-prefixes
(list "" "r" "R" "b" "B" "u" "U" "f" "F"
"rb" "rB" "Rb" "RB" "br" "bR" "Br" "BR"
"fr" "fR" "Fr" "FR" "rf" "rF" "Rf" "RF"))

(def %pp-string-rules
(fn (_)
(List flat-map
(fn (_ p)
(List flat-map
(fn (_ q)
(let ((q3 (Str8 append q q q)))
(list (Lexer until (lit string) (Str8 append p q3) q3 (lit to-end))
(Lexer nested (lit string) (Str8 append p q) (lit body)
(list (list (lit body) (%pp-byte q 0) 92 ())) (lit to-end)))))
(list "\"" "'")))
%pp-prefixes)))

; A number is digits, or a dot before a digit, then its body; the second
; pattern of each pair reads an exponent's sign, which the body alone would
; leave, and wins by length when there is one.
(def %pp-rules
(fn (_)
(List append
(list (Lexer run (lit blank) " \t\r\n" " \t\r\n")
(Lexer until (lit comment) "#" "\n" (lit to-end)))
(%pp-string-rules)
(list
(Lexer pattern (lit number)
(list (list %pp-digits 1 1) (list %pp-num-body 0 ())))
(Lexer pattern (lit number)
(list (list %pp-digits 1 1) (list %pp-num-mantissa 0 ()) (list "eE" 1 1)
(list "+-" 1 1) (list %pp-num-body 0 ())))
(Lexer pattern (lit number)
(list (list "." 1 1) (list %pp-digits 1 1) (list %pp-num-body 0 ())))
(Lexer pattern (lit number)
(list (list "." 1 1) (list %pp-digits 1 1) (list %pp-num-mantissa 0 ()) (list "eE" 1 1)
(list "+-" 1 1) (list %pp-num-body 0 ())))
(Lexer run (lit name) %pp-ident-start %pp-ident)
(Lexer pattern (lit decorator)
(list (list "@" 1 1) (list %pp-ident-start 1 1) (list %pp-ident 0 ())))
(Lexer any (lit plain))))))

; python/lexer's rules, the ones the highlighting grammar reads too.
(def %pp-lexer ())
(def %pp-lex
(fn (_)
(if (null? %pp-lexer) (set! %pp-lexer (Lexer make (%pp-rules))) ())
(if (null? %pp-lexer) (set! %pp-lexer (Lexer make (python-lexer-rules))) ())
%pp-lexer))

; Each tag's colour: the code strings above, read when a token is painted.
Expand Down
115 changes: 115 additions & 0 deletions python/syntax.x
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
; # x-python -- Python on x-lang
;
; ## python/syntax.x -- the Python highlighting grammar
;
; @description The grammar x/tool/syntax colours Python files with: the line
; editor's Lexer rules, with names classed as keywords, constants,
; builtins and classes.
; @author [Jon Ruttan](jonruttan@gmail.com)
; @copyright 2026 Jon Ruttan
; @license MIT No Attribution (MIT-0)
;
; lang.xon's (grammar python ...) form names this module, and x/tool/syntax
; imports it the first time a file needs the grammar, in whatever process is
; colouring: vi's, a REPL's, an HTML renderer's. So it imports x-lang
; libraries and python/lexer, never the interpreter, and the words it colours
; are its own copy of the interpreter's tables. A spec holds the two to the
; same words.
;
; A name is classed as the line editor's painter classes it: a keyword, then
; True, False and None as constants, then a builtin, then a leading capital
; as a class. A decorator is a keyword. A bracket, a comma, a colon, a
; semicolon or a dot is punctuation, every other plain byte an operator.

(module python/syntax)
(import x/type/list)
(import x/type/str)
(import x/tool/syntax)
(import python/lexer python-lexer-rules)

; The keywords python/tokens.x reads, as python-keywords lists them.
(def python-syntax-keywords
(list "if" "elif" "else" "while" "def" "return" "pass" "and" "or" "not"
"in" "is" "for" "break" "continue" "class" "import" "from" "as"
"try" "except" "finally" "raise" "with" "lambda" "global" "nonlocal"
"assert" "del" "yield" "async" "await"))

(def python-syntax-constants (list "True" "False" "None"))

; The builtins python/parse.x's python-builtins table names, each once, in
; the table's order, less the constants.
(def python-syntax-builtins
(list "print" "__debug__" "len" "range" "slice" "str" "repr" "list" "hasattr"
"object" "bytes" "staticmethod" "classmethod" "property" "type" "int"
"float" "bool" "dict" "tuple" "isinstance" "pow" "abs" "round" "min"
"max" "bytearray" "memoryview" "complex" "hash" "NotImplemented"
"Ellipsis" "eval" "exec" "compile" "dir" "globals" "locals" "vars" "chr"
"ord" "StopIteration" "StopAsyncIteration" "GeneratorExit" "next" "sum"
"map" "zip" "all" "any" "sorted" "iter" "set" "frozenset" "bin" "hex"
"oct" "divmod" "callable" "__import__" "__build_class__" "id" "getattr"
"setattr" "delattr" "issubclass" "enumerate" "filter" "reversed"
"SystemExit" "BaseException" "Exception" "ImportError"
"ModuleNotFoundError" "OSError" "FileNotFoundError" "PermissionError"
"FileExistsError" "IsADirectoryError" "open" "IOError" "EOFError"
"KeyboardInterrupt" "IndentationError" "UnicodeError"
"UnicodeDecodeError" "UnicodeEncodeError" "MemoryError" "OverflowError"
"NotImplementedError" "ArithmeticError" "LookupError"
"ZeroDivisionError" "IndexError" "KeyError" "AttributeError" "NameError"
"UnboundLocalError" "TypeError" "ValueError" "AssertionError"
"RuntimeError" "SyntaxError"))

(def %py-syntax-bref (prim-ref (lit str) (lit byte-ref)))
(def %py-syntax-cint (prim-ref (lit char) (lit ->int)))
(def %py-syntax-blen (prim-ref (lit str) (lit byte-len)))
(def %py-syntax-byte (fn (_ s i) (%py-syntax-cint (%py-syntax-bref s i))))
(def %py-syntax-upper? (fn (_ b) (if (>= b 65) (<= b 90) #f)))
(def %py-syntax-lower? (fn (_ b) (if (>= b 97) (<= b 122) #f)))

(def %py-syntax-name-class
(fn (_ text)
(match
((List includes? text python-syntax-keywords) (lit keyword))
((List includes? text python-syntax-constants) (lit constant))
((List includes? text python-syntax-builtins) (lit name))
((%py-syntax-upper? (%py-syntax-byte text 0)) (lit class))
(#t ()))))

; ( ) , . : ; [ ] { }
(def %py-syntax-punct (list 40 41 44 46 58 59 91 93 123 125))

; A blank is plain only at the text's end, where the blank run meets the
; Lexer's end text and is not a token; it has no class.
(def %py-syntax-plain-class
(fn (_ text)
(let ((b (%py-syntax-byte text 0)))
(match
((<= b 32) ())
((List includes? b %py-syntax-punct) (lit punct))
(#t (lit operator))))))

; A line starting with a name, a comment or a decorator at its first column
; is taken to start outside any string, so spans need not read the file from
; its top. A triple-quoted string with such a line in it colours inverted
; from that line to the end of the window, as other editors' Python grammars
; do.
(def %py-syntax-sync
(fn (_ s ls)
(if (>= ls (%py-syntax-blen s)) #f
(let ((b (%py-syntax-byte s ls)))
(match
((List includes? b (list 35 64 95)) #t)
((%py-syntax-upper? b) #t)
(#t (%py-syntax-lower? b)))))))

(Syntax register!
(let ((g (Syntax make (lit python) (python-lexer-rules)
(list (pair (lit comment) (lit comment))
(pair (lit string) (lit string))
(pair (lit number) (lit number))
(pair (lit decorator) (lit keyword))
(pair (lit name) %py-syntax-name-class)
(pair (lit plain) %py-syntax-plain-class)))))
(g sync %py-syntax-sync)
g))

(provide python/syntax python-syntax-keywords python-syntax-constants python-syntax-builtins)
Loading
Loading