diff --git a/Makefile b/Makefile index 157bf97..fe2e752 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ ISABELLE = isabelle +# The build targets delete their python_out/ first: export_files only adds and +# overwrites, so modules a build no longer produces would otherwise linger +# (and end up in test/diff/golden/ on the next reseed). PYTEST = uv run pytest PYTHON = uv run python3 @@ -12,37 +15,43 @@ test-slow: build-test-slow $(PYTHON) -m py_compile test/slow/generated_full/*.py build-test-quick: + rm -rf test/quick/python_out $(ISABELLE) build -d . -e -c Python_Test_Quick -test-quick: +test-quick: build-test-quick $(PYTEST) test/quick/run_test.py build-examples: + rm -rf examples/python_out $(ISABELLE) build -d . -e -c Python_Examples test-diff: build-examples $(PYTEST) test/diff/run_test.py build-test-oracle: + rm -rf test/oracle/python_out $(ISABELLE) build -d . -e -c Python_Test_Oracle test-oracle: build-test-oracle $(PYTEST) test/oracle/run_test.py +test-names: + $(PYTEST) test/test_unique_names.py + reseed: build-examples $(PYTHON) test/diff/reseed_goldens.py reserved-words: $(PYTHON) etc/update_reserved_words.py -test-all: test-quick test-slow test-diff test-oracle +test-all: test-quick test-slow test-diff test-oracle test-names test: test-quick build: $(ISABELLE) build -d . -e -c Python -.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle build reserved-words +.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle test-names build reserved-words diff --git a/ROOT b/ROOT index d2bdc1e..08c89aa 100644 --- a/ROOT +++ b/ROOT @@ -50,6 +50,7 @@ session Python_Test_Quick in "test/quick" = Python + theories [document = false] Quick_Test Quick_Test_Tail_Calls + Quick_Test_Name_Clash export_files (in "python_out") "*:**" session Python_Test_Slow in "test/slow" = "HOL-Library" + diff --git a/code_python.ML b/code_python.ML index 7730612..a5730b1 100644 --- a/code_python.ML +++ b/code_python.ML @@ -933,15 +933,18 @@ fun print_python_stmt undefineds const_syntax reserved args_num fun python_program_of_program ctxt module_name reserved identifiers exports program = let - (* Add trailing underscores to a name until it is unused and not a module name, following PEP 8 *) - fun namify_common module_names base ((nsp_class, nsp_object), nsp_common) = + (* Add trailing underscores to a name until it is not reserved, not a module name and not + used in this module yet, following PEP 8. The names used are kept in a plain set, not a + Name.context: Name.declare strips trailing underscores, so after declaring "insert_" the + context would still report "insert_" as free and hand it out a second time. *) + fun namify_common module_names base used = let - fun fresh x = - if Name.is_declared nsp_common x orelse member (op =) module_names x then fresh (x ^ "_") else x; + fun taken x = + Name.is_declared reserved x orelse Symtab.defined used x + orelse member (op =) module_names x; + fun fresh x = if taken x then fresh (x ^ "_") else x; val base' = fresh base; - in - (base', ((Name.declare base' nsp_class, Name.declare base' nsp_object), Name.declare base' nsp_common)) - end; + in (base', Symtab.insert_set base' used) end; (* Python doesn't really have any naming restrictions relating to upper/lowercase etc. *) fun namify_stmt _ (Code_Thingol.NoStmt) = raise Fail "should not be reached" @@ -961,7 +964,7 @@ fun python_program_of_program ctxt module_name reserved identifiers exports prog fun flat_program module_names = Code_Namespace.flat_program ctxt { module_prefix = "", module_name = module_name, reserved = reserved, - identifiers = identifiers, empty_nsp = ((reserved, reserved), reserved), + identifiers = identifiers, empty_nsp = Symtab.empty, namify_stmt = namify_stmt module_names, modify_stmt = modify_stmt } exports program; diff --git a/test/diff/golden/Python_Examples.Example_integer_bit_ops/Code_Numeral.py b/test/diff/golden/Python_Examples.Example_integer_bit_ops/Code_Numeral.py deleted file mode 100644 index 3eea699..0000000 --- a/test/diff/golden/Python_Examples.Example_integer_bit_ops/Code_Numeral.py +++ /dev/null @@ -1,62 +0,0 @@ -# Generated by Isabelle from theory HOL.Code_Numeral; do not edit - -from dataclasses import dataclass -from typing import Any, Callable -import functools -import Num -import Power -import Groups - -def _abort(msg, thunk): - raise RuntimeError(msg) - -@functools.cache -def one_integer(): - return Groups.one(one_ = lambda : 1) - -@functools.cache -def times_integer(): - return Groups.times(times_ = lambda a, b: (a * b)) - -@functools.cache -def power_integer(): - return Power.power(one_power = one_integer(), times_power = times_integer()) - -def drop_bit(i, k): - return ((lambda a, b: a // b if b else 0)(k, Power.power_(power_integer(), 2, (max(0, (abs(i))))))) - -def push_bit(i, k): - return ((k * Power.power_(power_integer(), 2, (max(0, (abs(i))))))) - -def sgn_integer(k): - if ((k == 0)): - return (0) - else: - if ((k < 0)): - return (-1) - else: - return (1) - -def push_bit_integer(n, k): - return (push_bit(n, k)) - -def bit_integer(k, n): - return (not ((((k & push_bit_integer(n, 1))) == 0))) - -def mask_integer(n): - return ((push_bit_integer(n, 1) - 1)) - -def set_bit_integer(n, k): - return ((k | push_bit_integer(n, 1))) - -def drop_bit_integer(n, k): - return (drop_bit(n, k)) - -def flip_bit_integer(n, k): - return ((k ^ push_bit_integer(n, 1))) - -def take_bit_integer(n, k): - return ((k & mask_integer(n))) - -def unset_bit_integer(n, k): - return ((k & ((~ push_bit_integer(n, 1))))) diff --git a/test/quick/Quick_Test_Name_Clash.thy b/test/quick/Quick_Test_Name_Clash.thy new file mode 100644 index 0000000..8c28d0f --- /dev/null +++ b/test/quick/Quick_Test_Name_Clash.thy @@ -0,0 +1,26 @@ +theory Quick_Test_Name_Clash + imports Main "Python.Python_Setup" +begin + +text \Three constants with the base name \insert\ in one Python module: \Set.insert\, + \List.insert\ (in the code equation of \Set.insert\ on \set xs\), and the one below. + Exported with \module_name\, so all three share a namespace and two of them need fresh + names. Each must get its own name: if two get the same one (e.g. both \insert_\), the later + \def\ shadows the earlier and calls to the first one run the second.\ + +definition insert :: "nat \ nat list \ nat list" where + "insert x xs = xs @ [x]" + +definition nc_own :: "nat \ nat list \ nat list" where + "nc_own x xs = insert x xs" + +definition nc_list :: "nat \ nat list \ nat list" where + "nc_list x xs = List.insert x xs" + +definition nc_set :: "nat \ nat \ nat list \ bool" where + "nc_set y x xs = (y \ Set.insert x (set xs))" + +export_code nc_own nc_list nc_set + in Python module_name Quick_Test_Name_Clash file_prefix "." + +end diff --git a/test/quick/run_test.py b/test/quick/run_test.py index 8cddd9b..0f8c8fe 100644 --- a/test/quick/run_test.py +++ b/test/quick/run_test.py @@ -327,3 +327,32 @@ def test_tail_calls_with_closures_stay_recursive(): tc = _load_tail_calls() assert tc.tc_shift_small == 13 assert tc.tc_shift(3, lambda x: x)(0) == 3 + + +# ---- name clashes in one module (Quick_Test_Name_Clash.thy, exported with module_name) ---- + +def _load_name_clash(): + matches = list(PYTHON_OUT_ROOT.rglob("Quick_Test_Name_Clash.py")) + if not matches: + pytest.skip("Quick_Test_Name_Clash.py not generated") + sys.path.insert(0, str(matches[0].parent)) + import Quick_Test_Name_Clash + return Quick_Test_Name_Clash + + +def test_name_clash_own_insert(): + nc = _load_name_clash() + assert nc.nc_own(3, [1, 2]) == [1, 2, 3] + + +def test_name_clash_list_insert(): + nc = _load_name_clash() + assert nc.nc_list(2, [1, 2]) == [1, 2] + assert nc.nc_list(3, [1, 2]) == [3, 1, 2] + + +def test_name_clash_set_insert(): + nc = _load_name_clash() + assert nc.nc_set(3, 1, [2, 3]) + assert nc.nc_set(1, 1, [2, 3]) + assert not nc.nc_set(4, 1, [2, 3]) diff --git a/test/test_unique_names.py b/test/test_unique_names.py new file mode 100644 index 0000000..04b39de --- /dev/null +++ b/test/test_unique_names.py @@ -0,0 +1,106 @@ +""" +Every generated module must bind each top-level name (and each name in a +class body) only once. If the backend gives two constants the same Python +name, the later binding silently shadows the earlier one and calls to the +first constant run the second; the code still imports and often still +passes tests that do not exercise both. + +The check is purely syntactic (ast), so it runs on whatever has been +generated: the outputs of the quick, oracle, slow and example sessions, and +the checked-in golden files. Build the sessions first (see the Makefile), +then: + + uv run pytest test/test_unique_names.py -q + +or, on any files or directories: + + uv run python3 test/test_unique_names.py PATH... +""" + +import ast +import sys +from collections import Counter +from pathlib import Path + +import pytest + +THIS_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = THIS_DIR.parent + +SEARCH_ROOTS = [ + PROJECT_ROOT / "examples" / "python_out", + THIS_DIR / "quick" / "python_out", + THIS_DIR / "oracle" / "python_out", + THIS_DIR / "slow" / "generated_basic", + THIS_DIR / "slow" / "generated_full", + THIS_DIR / "diff" / "golden", +] + +# Temporaries the backend invents (invent_name in code_python.ML) for lambdas and +# top-level matches; each is used right after it is bound, so rebinding is harmless. +TEMPORARIES = {"lam", "target"} + + +def _bound_names(body): + """Names bound directly by the statements of a module or class body.""" + for stmt in body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + yield stmt.name + elif isinstance(stmt, ast.Assign): + for target in stmt.targets: + if isinstance(target, ast.Name): + yield target.id + elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + yield stmt.target.id + elif isinstance(stmt, ast.TypeAlias): + yield stmt.name.id + + +def duplicate_names(source, filename=""): + """List of (scope, name, count) for every name bound more than once.""" + tree = ast.parse(source, filename) + scopes = [("", tree.body)] + scopes += [ + (stmt.name, stmt.body) for stmt in ast.walk(tree) if isinstance(stmt, ast.ClassDef) + ] + return [ + (scope, name, count) + for scope, body in scopes + for name, count in Counter(_bound_names(body)).items() + if count > 1 and name not in TEMPORARIES + ] + + +def _python_files(paths): + for path in paths: + if path.is_dir(): + yield from sorted(path.rglob("*.py")) + elif path.suffix == ".py": + yield path + + +_FILES = list(_python_files(root for root in SEARCH_ROOTS if root.exists())) + +if __name__ != "__main__" and not _FILES: + pytest.skip( + "No generated modules found -- build a test session first (e.g. " + "`isabelle build -d . -e Python_Test_Quick`).", + allow_module_level=True, + ) + + +@pytest.mark.parametrize("path", _FILES, ids=lambda p: p.relative_to(PROJECT_ROOT).as_posix()) +def test_names_bound_once(path): + dups = duplicate_names(path.read_text(), str(path)) + assert not dups, "names bound more than once: " + ", ".join( + f"{name} ({count}x in {scope})" for scope, name, count in dups + ) + + +if __name__ == "__main__": + status = 0 + for path in _python_files(Path(arg) for arg in sys.argv[1:]): + for scope, name, count in duplicate_names(path.read_text(), str(path)): + print(f"{path}: {name} bound {count} times in {scope}") + status = 1 + sys.exit(status)