Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,26 @@ jobs:
- name: Run diff test suite
run: uv run pytest test/diff/run_test.py

oracle:
runs-on: ubuntu-latest
container:
image: makarius/isabelle:Isabelle2025-2
options: --user root
steps:
- uses: actions/checkout@v4

- name: Install python3/pip and pytest
run: |
apt-get update
apt-get install -y --no-install-recommends python3 python3-pip
pip3 install --break-system-packages pytest 2>/dev/null || pip3 install pytest

- name: extract Python code
run: /home/isabelle/Isabelle/bin/isabelle build -v -d . -e Python_Test_Oracle

- name: Run oracle test suite
run: pytest test/oracle/run_test.py --tb=line

slow:
runs-on: ubuntu-latest
container:
Expand Down
10 changes: 8 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,23 @@ build-examples:
test-diff: build-examples
$(PYTEST) test/diff/run_test.py

build-test-oracle:
$(ISABELLE) build -d . -e Python_Test_Oracle

test-oracle: build-test-oracle
$(PYTEST) test/oracle/run_test.py

reseed: build-examples
$(PYTHON) test/diff/reseed_goldens.py

test-all: test-quick test-slow test-diff
test-all: test-quick test-slow test-diff test-oracle

test: test-quick

build:
$(ISABELLE) build -d . -e Python

.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build
.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle build



Expand Down
8 changes: 8 additions & 0 deletions ROOT
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,11 @@ session Python_Test_Slow in "test/slow" = "HOL-Library" +
Candidates
Extract_Basic
Extract_Full

session Python_Test_Oracle in "test/oracle" = Python +
description "Differential tests against Isabelle's SML evaluator"
options [timeout = 300, document = false]
theories
Oracle_Cases
Oracle_Cases_TC
export_files (in "python_out") "*:**"
82 changes: 82 additions & 0 deletions test/oracle/Oracle_Arith.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
theory Oracle_Arith
imports Python_Check "Python.Python_Setup"
begin

unbundle bit_operations_syntax

section \<open>div and mod on all four numeric types\<close>

python_check_grid int_div: "\<lambda>x y. (x::int) div y" over "[-7, -1, 0, 1, 7]" "[-2, 0, 2]"
python_check_grid int_mod: "\<lambda>x y. (x::int) mod y" over "[-7, -1, 0, 1, 7]" "[-2, 0, 2]"
python_check_grid nat_div: "\<lambda>x y. (x::nat) div y" over "[0, 1, 7]" "[0, 2, 3]"
python_check_grid nat_mod: "\<lambda>x y. (x::nat) mod y" over "[0, 1, 7]" "[0, 2, 3]"
python_check_grid integer_div: "\<lambda>x y. (x::integer) div y" over "[-7, 0, 7]" "[-2, 0, 2]"
python_check_grid integer_mod: "\<lambda>x y. (x::integer) mod y" over "[-7, 0, 7]" "[-2, 0, 2]"
python_check_grid natural_div: "\<lambda>x y. (x::natural) div y" over "[0, 7]" "[0, 2]"
python_check_grid natural_mod: "\<lambda>x y. (x::natural) mod y" over "[0, 7]" "[0, 2]"

text \<open>Negative control: truncating division would give \<open>-3\<close>.\<close>
python_check int_div_not_trunc: "(-7::int) div 2 \<noteq> -3"

section \<open>Subtraction and conversions into nat\<close>

python_check_grid nat_minus: "\<lambda>x y. (x::nat) - y" over "[0, 3, 10]" "[0, 3, 10]"
python_check_grid natural_minus: "\<lambda>x y. (x::natural) - y" over "[0, 3]" "[0, 3]"

definition to_nat :: "int \<Rightarrow> nat" where "to_nat x = nat x"

python_check nat_neg: "nat (-5)"
python_check nat_pos: "nat 5"
python_check to_nat_neg: "to_nat (-5)"
python_check_grid nat_of_diff: "\<lambda>x y. nat ((x::int) - y)" over "[0, 3]" "[0, 3]"
python_check int_nat_neg: "int (nat (-3)) + 1"
python_check nat_of_integer_neg: "nat_of_integer (-5)"
python_check natural_of_integer_neg: "natural_of_integer (-5)"
python_check int_of_nat: "int (3::nat) - 5"
python_check integer_of_int_ex: "integer_of_int (-5) * 2"
python_check int_of_integer_ex: "int_of_integer (-7) div 2"
python_check nat_of_natural_ex: "nat_of_natural 7 + 1"
python_check integer_of_nat_ex: "integer_of_nat 3 - 5"

section \<open>Big numbers and powers\<close>

python_check big_mul: "(2::int) ^ 100 * 3 ^ 50"
python_check big_div: "(- ((2::int) ^ 70)) div 3"
python_check big_mod: "(- ((2::int) ^ 70)) mod 7"
python_check big_literal: "(123456789012345678901234567890::int) + 1"
python_check big_neg_literal: "(-123456789012345678901234567890::int) * -1"
python_check big_nat: "(2::nat) ^ 80 - 2 ^ 81"
python_check_grid int_pow: "\<lambda>x n. (x::int) ^ n" over "[-2, 0, 3]" "[0, 1, 5]"
python_check neg_pow: "(-2::int) ^ 2"
python_check nat_pow_zero: "(0::nat) ^ 0"

section \<open>gcd, lcm, abs, sgn, min, max, comparisons\<close>

python_check_grid int_gcd: "\<lambda>x y. gcd (x::int) y" over "[-12, 0, 18]" "[-8, 0, 8]"
python_check_grid int_lcm: "\<lambda>x y. lcm (x::int) y" over "[-4, 0, 6]" "[-6, 0, 4]"
python_check_grid nat_gcd: "\<lambda>x y. gcd (x::nat) y" over "[0, 12]" "[0, 8]"
python_check_grid int_sgn: "\<lambda>x. sgn (x::int)" over "[-3, 0, 3]"
python_check_grid int_abs: "\<lambda>x. \<bar>x::int\<bar>" over "[-3, 0, 3]"
python_check_grid integer_abs: "\<lambda>x. \<bar>x::integer\<bar>" over "[-3, 0, 3]"
python_check_grid int_minmax: "\<lambda>x y. (min x y, max (x::int) y)" over "[-1, 2]" "[-1, 2]"
python_check_grid int_cmp: "\<lambda>x y. ((x::int) \<le> y, x < y, x = y)" over "[-1..1]" "[-1..1]"
python_check_grid int_even: "\<lambda>x. (even (x::int), odd x)" over "[-3, -2, 0, 7]"
python_check minus_assoc: "(5::int) - (3 - 1) - - 2"

section \<open>Bit operations\<close>

python_check_grid int_and: "\<lambda>x y. (x::int) AND y" over "[-6, -1, 0, 5]" "[-3, 0, 3]"
python_check_grid int_or: "\<lambda>x y. (x::int) OR y" over "[-6, -1, 0, 5]" "[-3, 0, 3]"
python_check_grid int_xor: "\<lambda>x y. (x::int) XOR y" over "[-6, -1, 0, 5]" "[-3, 0, 3]"
python_check_grid int_not: "\<lambda>x. NOT (x::int)" over "[-1, 0, 5]"
python_check_grid int_push_bit: "\<lambda>n x. push_bit n (x::int)" over "[0, 3]" "[-5, 5]"
python_check_grid int_drop_bit: "\<lambda>n x. drop_bit n (x::int)" over "[0, 1, 3]" "[-5, -1, 5]"
python_check_grid int_take_bit: "\<lambda>n x. take_bit n (x::int)" over "[0, 3]" "[-5, 5]"
python_check_grid int_signed_take_bit: "\<lambda>n x. signed_take_bit n (x::int)" over "[0, 3]" "[-5, 5, 6]"
python_check_grid int_bit: "\<lambda>x n. bit (x::int) n" over "[-5, 5]" "[0, 1, 10]"
python_check_grid int_mask: "\<lambda>n. mask n :: int" over "[0, 1, 8]"
python_check_grid nat_and: "\<lambda>x y. (x::nat) AND y" over "[0, 5, 12]" "[3, 10]"
python_check_grid nat_xor: "\<lambda>x y. (x::nat) XOR y" over "[0, 5, 12]" "[3, 10]"
python_check_grid nat_drop_bit: "\<lambda>n x. drop_bit n (x::nat)" over "[0, 2]" "[0, 13]"

end
15 changes: 15 additions & 0 deletions test/oracle/Oracle_Cases.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
theory Oracle_Cases
imports
Oracle_Arith
Oracle_Data
Oracle_Strings_Bools
Oracle_Classes
Oracle_Recursion
Oracle_Names
Oracle_Keywords
Oracle_Import_Cycle
begin

python_export_checks

end
17 changes: 17 additions & 0 deletions test/oracle/Oracle_Cases_TC.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
theory Oracle_Cases_TC
imports Oracle_Recursion
begin

text \<open>The recursion checks again, exported with self tail calls printed as loops,
plus inputs too deep for Python's default recursion limit of 1000.\<close>

declare [[python_tail_calls]]

python_check tc_build_long: "length (build 5000 [])"
python_check tc_rev_long: "length (acc_rev (build 5000 []) [])"
python_check tc_count_long: "count_pos (map int (build 5000 [])) 0"
python_check tc_swap_long: "swap_count 5001 1 2"

python_export_checks

end
61 changes: 61 additions & 0 deletions test/oracle/Oracle_Classes.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
theory Oracle_Classes
imports Python_Check "Python.Python_Setup"
begin

definition double_sum :: "'a::monoid_add list \<Rightarrow> 'a" where
"double_sum xs = sum_list xs + sum_list xs"

definition sorted_desc :: "'a::linorder list \<Rightarrow> 'a list" where
"sorted_desc xs = rev (sort xs)"

python_check c_ds_int: "double_sum [1::int, 2]"
python_check c_ds_nat: "double_sum [1::nat, 2]"
python_check c_ds_both: "(double_sum [1::int], double_sum [2::nat], double_sum [3::integer])"
python_check c_sort_bool: "sorted_desc [True, False, True]"
python_check c_fold_max: "fold max [3::int, 7, -2] 0"

class scalable =
fixes scale :: "int \<Rightarrow> 'a \<Rightarrow> 'a"

instantiation int :: scalable
begin
definition "scale k (x::int) = k * x"
instance ..
end

instantiation list :: (scalable) scalable
begin
definition "scale k xs = map (scale k) xs"
instance ..
end

instantiation prod :: (scalable, scalable) scalable
begin
definition "scale k p = (scale k (fst p), scale k (snd p))"
instance ..
end

class scalable_zero = scalable +
fixes szero :: 'a

instantiation int :: scalable_zero
begin
definition "szero = (0::int)"
instance ..
end

instantiation list :: (scalable) scalable_zero
begin
definition "szero = ([] :: 'a list)"
instance ..
end

definition sz_pair :: "'a::scalable_zero \<Rightarrow> 'a list" where
"sz_pair x = [scale 2 x, szero]"

python_check c_scale: "scale 3 ([(1::int, [2::int])], (4::int))"
python_check c_sz_int: "sz_pair (5::int)"
python_check c_sz_list: "sz_pair [5::int, 6]"
python_check c_sz_nested: "sz_pair [[5::int]]"

end
105 changes: 105 additions & 0 deletions test/oracle/Oracle_Data.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
theory Oracle_Data
imports Python_Check "Python.Python_Setup" "Python.Python_IArray"
begin

section \<open>Lists\<close>

fun alt :: "'a list \<Rightarrow> 'a list" where
"alt (x # y # zs) = x # alt zs"
| "alt xs = xs"

python_check l_rev: "rev [1::int, 2, 3]"
python_check l_sort: "sort [3::int, -1, 2, -1]"
python_check l_sort_key: "sort_key (\<lambda>x. - x) [3::int, -1, 2]"
python_check l_remdups: "remdups [1::nat, 2, 1, 3, 2]"
python_check l_zip: "zip [1::nat, 2, 3] [True, False]"
python_check l_upt: "[2..<6]"
python_check l_upt_empty: "[6..<2]"
python_check l_upt_long: "length [0..<5000]"
python_check l_upto: "[(-2::int)..2]"
python_check l_take_drop: "(take 2 [1::int, 2, 3], drop 5 [1::int, 2])"
python_check l_nth: "[10::int, 20, 30] ! 2"
python_check l_concat: "concat [[1::int], [], [2, 3]]"
python_check l_foldr: "foldr (\<lambda>x acc. x - acc) [1::int, 2, 3] 0"
python_check l_foldl: "foldl (\<lambda>acc x. acc - x) (0::int) [1, 2, 3]"
python_check l_sum: "sum_list [1::int, -2, 3]"
python_check l_sum_nat: "sum_list [1::nat, 2, 3]"
python_check l_prod: "prod_list [2::int, -3]"
python_check l_distinct: "(distinct [1::int, 2, 1], distinct [1::int, 2])"
python_check l_replicate: "replicate 3 (0::nat)"
python_check l_butlast: "butlast [1::int, 2, 3]"
python_check l_last: "last [1::int, 2, 3]"
python_check l_update: "[1::int, 2, 3][1 := 9]"
python_check l_filter: "filter (\<lambda>x. x mod 2 = 0) [1::int, 2, 3, 4]"
python_check l_find: "(find (\<lambda>x. x > 2) [1::int, 2, 3, 4], find (\<lambda>x. x > 9) [1::int])"
python_check l_append_nested: "([1::int] @ [2]) @ ([] @ [3])"
python_check l_cons_var: "let xs = [2::int, 3] in 1 # xs"
python_check l_cons_cons: "let xs = [3::int] in 1 # 2 # xs"
python_check l_length: "length [[1::int], [], [2]]"
python_check l_map_pair: "map (\<lambda>(a, b). a + b) [(1::int, 2), (3, 4)]"
python_check l_list_all: "(list_all (\<lambda>x. x > 0) [1::int, 2], list_ex (\<lambda>x. x < 0) [1::int])"
python_check l_insert: "(List.insert 2 [1::int, 3], List.insert 1 [1::int, 3])"
python_check l_product: "List.product [1::int, 2] [True, False]"
python_check l_enumerate: "enumerate 5 [True, False]"
python_check l_splice: "splice [1::int, 3] [2, 4, 6]"
python_check l_rotate: "rotate 2 [1::int, 2, 3]"
python_check l_min_list: "min_list [3::int, -1, 2]"
python_check l_maps: "List.maps (\<lambda>x. [x, x]) [1::int, 2]"
python_check l_nested_eq: "([[1::int], []] = [[1], []], [[1::int]] = [[]])"
python_check l_alt: "alt [1::int, 2, 3, 4, 5]"
python_check l_map_nested: "map (map (\<lambda>x. x + 1)) [[1::int], [], [2, 3]]"
python_check l_hd_tl: "(hd [1::int, 2], tl [1::int, 2])"
python_check l_member: "(List.member [1::int, 2] 2, (3::int) \<in> set [1, 2])"

section \<open>Options and pairs\<close>

python_check o_map: "map_option (\<lambda>x. x + 1) (Some (1::int))"
python_check o_bind: "Option.bind (Some (3::int)) (\<lambda>x. if x > 2 then None else Some x)"
python_check o_the: "the (Some (2::nat)) + 1"
python_check o_map_filter: "List.map_filter (\<lambda>x. if x > 1 then Some (x * 2) else None) [1::int, 2, 3]"
python_check o_case: "case (None::int option) of None \<Rightarrow> 0 | Some x \<Rightarrow> x"
python_check o_nested: "Some (Some (None::int option))"
python_check p_nested: "((1::int, True), (STR ''a'', [2::nat]))"
python_check p_swap: "prod.swap (1::int, False)"
python_check p_case: "case (3::int, 4::int) of (a, b) \<Rightarrow> a * b"
python_check p_case_prod: "case_prod (\<lambda>a b. a - b) (5::int, 7)"
python_check p_fst_snd: "fst (snd ((1::int), (2::int, 3::int)))"

section \<open>Sets\<close>

python_check s_union: "{1::int, 2} \<union> {2, 3}"
python_check s_eq: "{1::int, 2} = {2, 1, 1}"
python_check s_card: "card {1::int, 2, 2, 3}"
python_check s_max: "Max {3::int, -1, 2}"
python_check s_subset: "({1::int} \<subseteq> {1, 2}, {1::int, 2} \<subseteq> {1})"
python_check s_coset_mem: "(2::int) \<in> - {1}"
python_check s_coset_subset: "{1::int, 3} \<subseteq> - {2}"
python_check s_image: "(\<lambda>x. x mod 2) ` {1::int, 2, 3}"
python_check s_filter: "Set.filter (\<lambda>x. x > 1) {1::int, 2, 3}"
python_check s_inter_coset: "{1::int, 2, 3} \<inter> - {2}"
python_check s_min: "min {1::int} {2}"
python_check s_max_sets: "max {1::int} {2}"
python_check s_of_sets: "card {{1::int}, {1}, {2}}"

section \<open>Records\<close>

record point = px :: int py :: int
record cpoint = point + col :: bool

python_check r_update: "px (\<lparr>px = 1, py = 2\<rparr>\<lparr>px := 5\<rparr>)"
python_check r_eq: "(\<lparr>px = 1, py = 2\<rparr> = \<lparr>px = 1, py = (2::int)\<rparr>, \<lparr>px = 1, py = 2\<rparr> = \<lparr>px = 2, py = (1::int)\<rparr>)"
python_check r_ext: "col (\<lparr>px = 1, py = 2, col = True\<rparr>\<lparr>py := 3\<rparr>)"
python_check r_ext_px: "px (\<lparr>px = 1, py = 2, col = True\<rparr>\<lparr>py := 3\<rparr>)"

section \<open>Immutable arrays\<close>

python_check ia_sub: "IArray [1::int, 2, 3] !! 1"
python_check ia_len: "IArray.length (IArray [True, False])"
python_check ia_of_fun: "IArray.list_of (IArray.of_fun (\<lambda>i. i * i) 4)"
python_check ia_eq: "(IArray [1::int] = IArray [1], IArray [1::int] = IArray [])"
python_check ia_all: "(IArray.all (\<lambda>x. x > 0) (IArray [1::int, 2]), IArray.exists (\<lambda>x. x > 1) (IArray [1::int]))"
python_check ia_tabulate: "IArray.list_of (IArray.tabulate (3, \<lambda>i. i + 1))"
python_check ia_sub_pair: "IArray.sub' (IArray [10::int, 20], 1)"
python_check ia_sub_pair_var: "let p = (IArray [10::int, 20], 1) in IArray.sub' p"

end
12 changes: 12 additions & 0 deletions test/oracle/Oracle_Import_Cycle.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
theory Oracle_Import_Cycle
imports Python_Check "Python.Python_Setup"
begin

text \<open>Sorting string literals makes this module import \<open>String\<close> before \<open>Arith\<close>.
The generated \<open>Arith\<close> and \<open>String\<close> import each other, and \<open>Arith\<close> uses
\<open>String.char_of_integer\<close> at import time. Kept apart because the failure is at
import, so it fails every check in its module.\<close>

python_check sort_str: "sort [STR ''b'', STR ''a'', STR ''c'']"

end
16 changes: 16 additions & 0 deletions test/oracle/Oracle_Keywords.thy
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
theory Oracle_Keywords
imports Python_Check "Python.Python_Setup"
begin

text \<open>Names that are Python keywords. Kept in their own module: if one is not
renamed, the module does not even parse.\<close>

definition async :: int where "async = 1"
definition await :: int where "await = 2"
definition k_vars :: "int \<Rightarrow> int \<Rightarrow> int \<Rightarrow> int" where
"k_vars nonlocal global del = nonlocal + global + del"

python_check k_async: "async + await"
python_check k_vars: "k_vars 1 2 3"

end
Loading
Loading