Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
ISABELLE = isabelle
# The build targets delete their python_out/ first: export_files only adds and
# overwrites, so modules a build no longer produces would otherwise linger
# (and end up in test/diff/golden/ on the next reseed).
PYTEST = uv run pytest
PYTHON = uv run python3

Expand All @@ -12,37 +15,43 @@ test-slow: build-test-slow
$(PYTHON) -m py_compile test/slow/generated_full/*.py

build-test-quick:
rm -rf test/quick/python_out
$(ISABELLE) build -d . -e -c Python_Test_Quick

test-quick:
test-quick: build-test-quick
$(PYTEST) test/quick/run_test.py

build-examples:
rm -rf examples/python_out
$(ISABELLE) build -d . -e -c Python_Examples

test-diff: build-examples
$(PYTEST) test/diff/run_test.py

build-test-oracle:
rm -rf test/oracle/python_out
$(ISABELLE) build -d . -e -c Python_Test_Oracle

test-oracle: build-test-oracle
$(PYTEST) test/oracle/run_test.py

test-names:
$(PYTEST) test/test_unique_names.py

reseed: build-examples
$(PYTHON) test/diff/reseed_goldens.py

reserved-words:
$(PYTHON) etc/update_reserved_words.py

test-all: test-quick test-slow test-diff test-oracle
test-all: test-quick test-slow test-diff test-oracle test-names

test: test-quick

build:
$(ISABELLE) build -d . -e -c Python

.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle build reserved-words
.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle test-names build reserved-words



Expand Down
1 change: 1 addition & 0 deletions ROOT
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ session Python_Test_Quick in "test/quick" = Python +
theories [document = false]
Quick_Test
Quick_Test_Tail_Calls
Quick_Test_Name_Clash
export_files (in "python_out") "*:**"

session Python_Test_Slow in "test/slow" = "HOL-Library" +
Expand Down
19 changes: 11 additions & 8 deletions code_python.ML
Original file line number Diff line number Diff line change
Expand Up @@ -933,15 +933,18 @@ fun print_python_stmt undefineds const_syntax reserved args_num

fun python_program_of_program ctxt module_name reserved identifiers exports program =
let
(* Add trailing underscores to a name until it is unused and not a module name, following PEP 8 *)
fun namify_common module_names base ((nsp_class, nsp_object), nsp_common) =
(* Add trailing underscores to a name until it is not reserved, not a module name and not
used in this module yet, following PEP 8. The names used are kept in a plain set, not a
Name.context: Name.declare strips trailing underscores, so after declaring "insert_" the
context would still report "insert_" as free and hand it out a second time. *)
fun namify_common module_names base used =
let
fun fresh x =
if Name.is_declared nsp_common x orelse member (op =) module_names x then fresh (x ^ "_") else x;
fun taken x =
Name.is_declared reserved x orelse Symtab.defined used x
orelse member (op =) module_names x;
fun fresh x = if taken x then fresh (x ^ "_") else x;
val base' = fresh base;
in
(base', ((Name.declare base' nsp_class, Name.declare base' nsp_object), Name.declare base' nsp_common))
end;
in (base', Symtab.insert_set base' used) end;

(* Python doesn't really have any naming restrictions relating to upper/lowercase etc. *)
fun namify_stmt _ (Code_Thingol.NoStmt) = raise Fail "should not be reached"
Expand All @@ -961,7 +964,7 @@ fun python_program_of_program ctxt module_name reserved identifiers exports prog
fun flat_program module_names =
Code_Namespace.flat_program ctxt
{ module_prefix = "", module_name = module_name, reserved = reserved,
identifiers = identifiers, empty_nsp = ((reserved, reserved), reserved),
identifiers = identifiers, empty_nsp = Symtab.empty,
namify_stmt = namify_stmt module_names, modify_stmt = modify_stmt }
exports program;

Expand Down

This file was deleted.

26 changes: 26 additions & 0 deletions test/quick/Quick_Test_Name_Clash.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
theory Quick_Test_Name_Clash
imports Main "Python.Python_Setup"
begin

text \<open>Three constants with the base name \<open>insert\<close> in one Python module: \<open>Set.insert\<close>,
\<open>List.insert\<close> (in the code equation of \<open>Set.insert\<close> on \<open>set xs\<close>), and the one below.
Exported with \<open>module_name\<close>, so all three share a namespace and two of them need fresh
names. Each must get its own name: if two get the same one (e.g. both \<open>insert_\<close>), the later
\<open>def\<close> shadows the earlier and calls to the first one run the second.\<close>

definition insert :: "nat \<Rightarrow> nat list \<Rightarrow> nat list" where
"insert x xs = xs @ [x]"

definition nc_own :: "nat \<Rightarrow> nat list \<Rightarrow> nat list" where
"nc_own x xs = insert x xs"

definition nc_list :: "nat \<Rightarrow> nat list \<Rightarrow> nat list" where
"nc_list x xs = List.insert x xs"

definition nc_set :: "nat \<Rightarrow> nat \<Rightarrow> nat list \<Rightarrow> bool" where
"nc_set y x xs = (y \<in> Set.insert x (set xs))"

export_code nc_own nc_list nc_set
in Python module_name Quick_Test_Name_Clash file_prefix "."

end
29 changes: 29 additions & 0 deletions test/quick/run_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,3 +327,32 @@ def test_tail_calls_with_closures_stay_recursive():
tc = _load_tail_calls()
assert tc.tc_shift_small == 13
assert tc.tc_shift(3, lambda x: x)(0) == 3


# ---- name clashes in one module (Quick_Test_Name_Clash.thy, exported with module_name) ----

def _load_name_clash():
matches = list(PYTHON_OUT_ROOT.rglob("Quick_Test_Name_Clash.py"))
if not matches:
pytest.skip("Quick_Test_Name_Clash.py not generated")
sys.path.insert(0, str(matches[0].parent))
import Quick_Test_Name_Clash
return Quick_Test_Name_Clash


def test_name_clash_own_insert():
nc = _load_name_clash()
assert nc.nc_own(3, [1, 2]) == [1, 2, 3]


def test_name_clash_list_insert():
nc = _load_name_clash()
assert nc.nc_list(2, [1, 2]) == [1, 2]
assert nc.nc_list(3, [1, 2]) == [3, 1, 2]


def test_name_clash_set_insert():
nc = _load_name_clash()
assert nc.nc_set(3, 1, [2, 3])
assert nc.nc_set(1, 1, [2, 3])
assert not nc.nc_set(4, 1, [2, 3])
106 changes: 106 additions & 0 deletions test/test_unique_names.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
"""
Every generated module must bind each top-level name (and each name in a
class body) only once. If the backend gives two constants the same Python
name, the later binding silently shadows the earlier one and calls to the
first constant run the second; the code still imports and often still
passes tests that do not exercise both.

The check is purely syntactic (ast), so it runs on whatever has been
generated: the outputs of the quick, oracle, slow and example sessions, and
the checked-in golden files. Build the sessions first (see the Makefile),
then:

uv run pytest test/test_unique_names.py -q

or, on any files or directories:

uv run python3 test/test_unique_names.py PATH...
"""

import ast
import sys
from collections import Counter
from pathlib import Path

import pytest

THIS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = THIS_DIR.parent

SEARCH_ROOTS = [
PROJECT_ROOT / "examples" / "python_out",
THIS_DIR / "quick" / "python_out",
THIS_DIR / "oracle" / "python_out",
THIS_DIR / "slow" / "generated_basic",
THIS_DIR / "slow" / "generated_full",
THIS_DIR / "diff" / "golden",
]

# Temporaries the backend invents (invent_name in code_python.ML) for lambdas and
# top-level matches; each is used right after it is bound, so rebinding is harmless.
TEMPORARIES = {"lam", "target"}


def _bound_names(body):
"""Names bound directly by the statements of a module or class body."""
for stmt in body:
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
yield stmt.name
elif isinstance(stmt, ast.Assign):
for target in stmt.targets:
if isinstance(target, ast.Name):
yield target.id
elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
yield stmt.target.id
elif isinstance(stmt, ast.TypeAlias):
yield stmt.name.id


def duplicate_names(source, filename="<generated>"):
"""List of (scope, name, count) for every name bound more than once."""
tree = ast.parse(source, filename)
scopes = [("<module>", tree.body)]
scopes += [
(stmt.name, stmt.body) for stmt in ast.walk(tree) if isinstance(stmt, ast.ClassDef)
]
return [
(scope, name, count)
for scope, body in scopes
for name, count in Counter(_bound_names(body)).items()
if count > 1 and name not in TEMPORARIES
]


def _python_files(paths):
for path in paths:
if path.is_dir():
yield from sorted(path.rglob("*.py"))
elif path.suffix == ".py":
yield path


_FILES = list(_python_files(root for root in SEARCH_ROOTS if root.exists()))

if __name__ != "__main__" and not _FILES:
pytest.skip(
"No generated modules found -- build a test session first (e.g. "
"`isabelle build -d . -e Python_Test_Quick`).",
allow_module_level=True,
)


@pytest.mark.parametrize("path", _FILES, ids=lambda p: p.relative_to(PROJECT_ROOT).as_posix())
def test_names_bound_once(path):
dups = duplicate_names(path.read_text(), str(path))
assert not dups, "names bound more than once: " + ", ".join(
f"{name} ({count}x in {scope})" for scope, name, count in dups
)


if __name__ == "__main__":
status = 0
for path in _python_files(Path(arg) for arg in sys.argv[1:]):
for scope, name, count in duplicate_names(path.read_text(), str(path)):
print(f"{path}: {name} bound {count} times in {scope}")
status = 1
sys.exit(status)
Loading