diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72a8db7..df12e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,26 @@ jobs: - name: Run diff test suite run: uv run pytest test/diff/run_test.py + oracle: + runs-on: ubuntu-latest + container: + image: makarius/isabelle:Isabelle2025-2 + options: --user root + steps: + - uses: actions/checkout@v4 + + - name: Install python3/pip and pytest + run: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install --break-system-packages pytest 2>/dev/null || pip3 install pytest + + - name: extract Python code + run: /home/isabelle/Isabelle/bin/isabelle build -v -d . -e Python_Test_Oracle + + - name: Run oracle test suite + run: pytest test/oracle/run_test.py --tb=line + slow: runs-on: ubuntu-latest container: diff --git a/Makefile b/Makefile index f545136..590ae11 100644 --- a/Makefile +++ b/Makefile @@ -23,17 +23,23 @@ build-examples: test-diff: build-examples $(PYTEST) test/diff/run_test.py +build-test-oracle: + $(ISABELLE) build -d . -e Python_Test_Oracle + +test-oracle: build-test-oracle + $(PYTEST) test/oracle/run_test.py + reseed: build-examples $(PYTHON) test/diff/reseed_goldens.py -test-all: test-quick test-slow test-diff +test-all: test-quick test-slow test-diff test-oracle test: test-quick build: $(ISABELLE) build -d . -e Python -.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build +.PHONY: default build-test-slow test-slow build-test-quick test-quick build-examples test-diff reseed test-all test build-test-oracle test-oracle build diff --git a/ROOT b/ROOT index 593f132..963b860 100644 --- a/ROOT +++ b/ROOT @@ -64,3 +64,11 @@ session Python_Test_Slow in "test/slow" = "HOL-Library" + Candidates Extract_Basic Extract_Full + +session Python_Test_Oracle in "test/oracle" = Python + + description "Differential tests against Isabelle's SML evaluator" + options [timeout = 300, document = false] + theories + Oracle_Cases + Oracle_Cases_TC + export_files (in "python_out") "*:**" diff --git a/test/oracle/Oracle_Arith.thy b/test/oracle/Oracle_Arith.thy new file mode 100644 index 0000000..ffa2f73 --- /dev/null +++ b/test/oracle/Oracle_Arith.thy @@ -0,0 +1,82 @@ +theory Oracle_Arith + imports Python_Check "Python.Python_Setup" +begin + +unbundle bit_operations_syntax + +section \div and mod on all four numeric types\ + +python_check_grid int_div: "\x y. (x::int) div y" over "[-7, -1, 0, 1, 7]" "[-2, 0, 2]" +python_check_grid int_mod: "\x y. (x::int) mod y" over "[-7, -1, 0, 1, 7]" "[-2, 0, 2]" +python_check_grid nat_div: "\x y. (x::nat) div y" over "[0, 1, 7]" "[0, 2, 3]" +python_check_grid nat_mod: "\x y. (x::nat) mod y" over "[0, 1, 7]" "[0, 2, 3]" +python_check_grid integer_div: "\x y. (x::integer) div y" over "[-7, 0, 7]" "[-2, 0, 2]" +python_check_grid integer_mod: "\x y. (x::integer) mod y" over "[-7, 0, 7]" "[-2, 0, 2]" +python_check_grid natural_div: "\x y. (x::natural) div y" over "[0, 7]" "[0, 2]" +python_check_grid natural_mod: "\x y. (x::natural) mod y" over "[0, 7]" "[0, 2]" + +text \Negative control: truncating division would give \-3\.\ +python_check int_div_not_trunc: "(-7::int) div 2 \ -3" + +section \Subtraction and conversions into nat\ + +python_check_grid nat_minus: "\x y. (x::nat) - y" over "[0, 3, 10]" "[0, 3, 10]" +python_check_grid natural_minus: "\x y. (x::natural) - y" over "[0, 3]" "[0, 3]" + +definition to_nat :: "int \ nat" where "to_nat x = nat x" + +python_check nat_neg: "nat (-5)" +python_check nat_pos: "nat 5" +python_check to_nat_neg: "to_nat (-5)" +python_check_grid nat_of_diff: "\x y. nat ((x::int) - y)" over "[0, 3]" "[0, 3]" +python_check int_nat_neg: "int (nat (-3)) + 1" +python_check nat_of_integer_neg: "nat_of_integer (-5)" +python_check natural_of_integer_neg: "natural_of_integer (-5)" +python_check int_of_nat: "int (3::nat) - 5" +python_check integer_of_int_ex: "integer_of_int (-5) * 2" +python_check int_of_integer_ex: "int_of_integer (-7) div 2" +python_check nat_of_natural_ex: "nat_of_natural 7 + 1" +python_check integer_of_nat_ex: "integer_of_nat 3 - 5" + +section \Big numbers and powers\ + +python_check big_mul: "(2::int) ^ 100 * 3 ^ 50" +python_check big_div: "(- ((2::int) ^ 70)) div 3" +python_check big_mod: "(- ((2::int) ^ 70)) mod 7" +python_check big_literal: "(123456789012345678901234567890::int) + 1" +python_check big_neg_literal: "(-123456789012345678901234567890::int) * -1" +python_check big_nat: "(2::nat) ^ 80 - 2 ^ 81" +python_check_grid int_pow: "\x n. (x::int) ^ n" over "[-2, 0, 3]" "[0, 1, 5]" +python_check neg_pow: "(-2::int) ^ 2" +python_check nat_pow_zero: "(0::nat) ^ 0" + +section \gcd, lcm, abs, sgn, min, max, comparisons\ + +python_check_grid int_gcd: "\x y. gcd (x::int) y" over "[-12, 0, 18]" "[-8, 0, 8]" +python_check_grid int_lcm: "\x y. lcm (x::int) y" over "[-4, 0, 6]" "[-6, 0, 4]" +python_check_grid nat_gcd: "\x y. gcd (x::nat) y" over "[0, 12]" "[0, 8]" +python_check_grid int_sgn: "\x. sgn (x::int)" over "[-3, 0, 3]" +python_check_grid int_abs: "\x. \x::int\" over "[-3, 0, 3]" +python_check_grid integer_abs: "\x. \x::integer\" over "[-3, 0, 3]" +python_check_grid int_minmax: "\x y. (min x y, max (x::int) y)" over "[-1, 2]" "[-1, 2]" +python_check_grid int_cmp: "\x y. ((x::int) \ y, x < y, x = y)" over "[-1..1]" "[-1..1]" +python_check_grid int_even: "\x. (even (x::int), odd x)" over "[-3, -2, 0, 7]" +python_check minus_assoc: "(5::int) - (3 - 1) - - 2" + +section \Bit operations\ + +python_check_grid int_and: "\x y. (x::int) AND y" over "[-6, -1, 0, 5]" "[-3, 0, 3]" +python_check_grid int_or: "\x y. (x::int) OR y" over "[-6, -1, 0, 5]" "[-3, 0, 3]" +python_check_grid int_xor: "\x y. (x::int) XOR y" over "[-6, -1, 0, 5]" "[-3, 0, 3]" +python_check_grid int_not: "\x. NOT (x::int)" over "[-1, 0, 5]" +python_check_grid int_push_bit: "\n x. push_bit n (x::int)" over "[0, 3]" "[-5, 5]" +python_check_grid int_drop_bit: "\n x. drop_bit n (x::int)" over "[0, 1, 3]" "[-5, -1, 5]" +python_check_grid int_take_bit: "\n x. take_bit n (x::int)" over "[0, 3]" "[-5, 5]" +python_check_grid int_signed_take_bit: "\n x. signed_take_bit n (x::int)" over "[0, 3]" "[-5, 5, 6]" +python_check_grid int_bit: "\x n. bit (x::int) n" over "[-5, 5]" "[0, 1, 10]" +python_check_grid int_mask: "\n. mask n :: int" over "[0, 1, 8]" +python_check_grid nat_and: "\x y. (x::nat) AND y" over "[0, 5, 12]" "[3, 10]" +python_check_grid nat_xor: "\x y. (x::nat) XOR y" over "[0, 5, 12]" "[3, 10]" +python_check_grid nat_drop_bit: "\n x. drop_bit n (x::nat)" over "[0, 2]" "[0, 13]" + +end diff --git a/test/oracle/Oracle_Cases.thy b/test/oracle/Oracle_Cases.thy new file mode 100644 index 0000000..73ff7b5 --- /dev/null +++ b/test/oracle/Oracle_Cases.thy @@ -0,0 +1,15 @@ +theory Oracle_Cases + imports + Oracle_Arith + Oracle_Data + Oracle_Strings_Bools + Oracle_Classes + Oracle_Recursion + Oracle_Names + Oracle_Keywords + Oracle_Import_Cycle +begin + +python_export_checks + +end diff --git a/test/oracle/Oracle_Cases_TC.thy b/test/oracle/Oracle_Cases_TC.thy new file mode 100644 index 0000000..8267ed6 --- /dev/null +++ b/test/oracle/Oracle_Cases_TC.thy @@ -0,0 +1,17 @@ +theory Oracle_Cases_TC + imports Oracle_Recursion +begin + +text \The recursion checks again, exported with self tail calls printed as loops, + plus inputs too deep for Python's default recursion limit of 1000.\ + +declare [[python_tail_calls]] + +python_check tc_build_long: "length (build 5000 [])" +python_check tc_rev_long: "length (acc_rev (build 5000 []) [])" +python_check tc_count_long: "count_pos (map int (build 5000 [])) 0" +python_check tc_swap_long: "swap_count 5001 1 2" + +python_export_checks + +end diff --git a/test/oracle/Oracle_Classes.thy b/test/oracle/Oracle_Classes.thy new file mode 100644 index 0000000..3c2d69e --- /dev/null +++ b/test/oracle/Oracle_Classes.thy @@ -0,0 +1,61 @@ +theory Oracle_Classes + imports Python_Check "Python.Python_Setup" +begin + +definition double_sum :: "'a::monoid_add list \ 'a" where + "double_sum xs = sum_list xs + sum_list xs" + +definition sorted_desc :: "'a::linorder list \ 'a list" where + "sorted_desc xs = rev (sort xs)" + +python_check c_ds_int: "double_sum [1::int, 2]" +python_check c_ds_nat: "double_sum [1::nat, 2]" +python_check c_ds_both: "(double_sum [1::int], double_sum [2::nat], double_sum [3::integer])" +python_check c_sort_bool: "sorted_desc [True, False, True]" +python_check c_fold_max: "fold max [3::int, 7, -2] 0" + +class scalable = + fixes scale :: "int \ 'a \ 'a" + +instantiation int :: scalable +begin +definition "scale k (x::int) = k * x" +instance .. +end + +instantiation list :: (scalable) scalable +begin +definition "scale k xs = map (scale k) xs" +instance .. +end + +instantiation prod :: (scalable, scalable) scalable +begin +definition "scale k p = (scale k (fst p), scale k (snd p))" +instance .. +end + +class scalable_zero = scalable + + fixes szero :: 'a + +instantiation int :: scalable_zero +begin +definition "szero = (0::int)" +instance .. +end + +instantiation list :: (scalable) scalable_zero +begin +definition "szero = ([] :: 'a list)" +instance .. +end + +definition sz_pair :: "'a::scalable_zero \ 'a list" where + "sz_pair x = [scale 2 x, szero]" + +python_check c_scale: "scale 3 ([(1::int, [2::int])], (4::int))" +python_check c_sz_int: "sz_pair (5::int)" +python_check c_sz_list: "sz_pair [5::int, 6]" +python_check c_sz_nested: "sz_pair [[5::int]]" + +end diff --git a/test/oracle/Oracle_Data.thy b/test/oracle/Oracle_Data.thy new file mode 100644 index 0000000..ebaba0b --- /dev/null +++ b/test/oracle/Oracle_Data.thy @@ -0,0 +1,105 @@ +theory Oracle_Data + imports Python_Check "Python.Python_Setup" "Python.Python_IArray" +begin + +section \Lists\ + +fun alt :: "'a list \ 'a list" where + "alt (x # y # zs) = x # alt zs" +| "alt xs = xs" + +python_check l_rev: "rev [1::int, 2, 3]" +python_check l_sort: "sort [3::int, -1, 2, -1]" +python_check l_sort_key: "sort_key (\x. - x) [3::int, -1, 2]" +python_check l_remdups: "remdups [1::nat, 2, 1, 3, 2]" +python_check l_zip: "zip [1::nat, 2, 3] [True, False]" +python_check l_upt: "[2..<6]" +python_check l_upt_empty: "[6..<2]" +python_check l_upt_long: "length [0..<5000]" +python_check l_upto: "[(-2::int)..2]" +python_check l_take_drop: "(take 2 [1::int, 2, 3], drop 5 [1::int, 2])" +python_check l_nth: "[10::int, 20, 30] ! 2" +python_check l_concat: "concat [[1::int], [], [2, 3]]" +python_check l_foldr: "foldr (\x acc. x - acc) [1::int, 2, 3] 0" +python_check l_foldl: "foldl (\acc x. acc - x) (0::int) [1, 2, 3]" +python_check l_sum: "sum_list [1::int, -2, 3]" +python_check l_sum_nat: "sum_list [1::nat, 2, 3]" +python_check l_prod: "prod_list [2::int, -3]" +python_check l_distinct: "(distinct [1::int, 2, 1], distinct [1::int, 2])" +python_check l_replicate: "replicate 3 (0::nat)" +python_check l_butlast: "butlast [1::int, 2, 3]" +python_check l_last: "last [1::int, 2, 3]" +python_check l_update: "[1::int, 2, 3][1 := 9]" +python_check l_filter: "filter (\x. x mod 2 = 0) [1::int, 2, 3, 4]" +python_check l_find: "(find (\x. x > 2) [1::int, 2, 3, 4], find (\x. x > 9) [1::int])" +python_check l_append_nested: "([1::int] @ [2]) @ ([] @ [3])" +python_check l_cons_var: "let xs = [2::int, 3] in 1 # xs" +python_check l_cons_cons: "let xs = [3::int] in 1 # 2 # xs" +python_check l_length: "length [[1::int], [], [2]]" +python_check l_map_pair: "map (\(a, b). a + b) [(1::int, 2), (3, 4)]" +python_check l_list_all: "(list_all (\x. x > 0) [1::int, 2], list_ex (\x. x < 0) [1::int])" +python_check l_insert: "(List.insert 2 [1::int, 3], List.insert 1 [1::int, 3])" +python_check l_product: "List.product [1::int, 2] [True, False]" +python_check l_enumerate: "enumerate 5 [True, False]" +python_check l_splice: "splice [1::int, 3] [2, 4, 6]" +python_check l_rotate: "rotate 2 [1::int, 2, 3]" +python_check l_min_list: "min_list [3::int, -1, 2]" +python_check l_maps: "List.maps (\x. [x, x]) [1::int, 2]" +python_check l_nested_eq: "([[1::int], []] = [[1], []], [[1::int]] = [[]])" +python_check l_alt: "alt [1::int, 2, 3, 4, 5]" +python_check l_map_nested: "map (map (\x. x + 1)) [[1::int], [], [2, 3]]" +python_check l_hd_tl: "(hd [1::int, 2], tl [1::int, 2])" +python_check l_member: "(List.member [1::int, 2] 2, (3::int) \ set [1, 2])" + +section \Options and pairs\ + +python_check o_map: "map_option (\x. x + 1) (Some (1::int))" +python_check o_bind: "Option.bind (Some (3::int)) (\x. if x > 2 then None else Some x)" +python_check o_the: "the (Some (2::nat)) + 1" +python_check o_map_filter: "List.map_filter (\x. if x > 1 then Some (x * 2) else None) [1::int, 2, 3]" +python_check o_case: "case (None::int option) of None \ 0 | Some x \ x" +python_check o_nested: "Some (Some (None::int option))" +python_check p_nested: "((1::int, True), (STR ''a'', [2::nat]))" +python_check p_swap: "prod.swap (1::int, False)" +python_check p_case: "case (3::int, 4::int) of (a, b) \ a * b" +python_check p_case_prod: "case_prod (\a b. a - b) (5::int, 7)" +python_check p_fst_snd: "fst (snd ((1::int), (2::int, 3::int)))" + +section \Sets\ + +python_check s_union: "{1::int, 2} \ {2, 3}" +python_check s_eq: "{1::int, 2} = {2, 1, 1}" +python_check s_card: "card {1::int, 2, 2, 3}" +python_check s_max: "Max {3::int, -1, 2}" +python_check s_subset: "({1::int} \ {1, 2}, {1::int, 2} \ {1})" +python_check s_coset_mem: "(2::int) \ - {1}" +python_check s_coset_subset: "{1::int, 3} \ - {2}" +python_check s_image: "(\x. x mod 2) ` {1::int, 2, 3}" +python_check s_filter: "Set.filter (\x. x > 1) {1::int, 2, 3}" +python_check s_inter_coset: "{1::int, 2, 3} \ - {2}" +python_check s_min: "min {1::int} {2}" +python_check s_max_sets: "max {1::int} {2}" +python_check s_of_sets: "card {{1::int}, {1}, {2}}" + +section \Records\ + +record point = px :: int py :: int +record cpoint = point + col :: bool + +python_check r_update: "px (\px = 1, py = 2\\px := 5\)" +python_check r_eq: "(\px = 1, py = 2\ = \px = 1, py = (2::int)\, \px = 1, py = 2\ = \px = 2, py = (1::int)\)" +python_check r_ext: "col (\px = 1, py = 2, col = True\\py := 3\)" +python_check r_ext_px: "px (\px = 1, py = 2, col = True\\py := 3\)" + +section \Immutable arrays\ + +python_check ia_sub: "IArray [1::int, 2, 3] !! 1" +python_check ia_len: "IArray.length (IArray [True, False])" +python_check ia_of_fun: "IArray.list_of (IArray.of_fun (\i. i * i) 4)" +python_check ia_eq: "(IArray [1::int] = IArray [1], IArray [1::int] = IArray [])" +python_check ia_all: "(IArray.all (\x. x > 0) (IArray [1::int, 2]), IArray.exists (\x. x > 1) (IArray [1::int]))" +python_check ia_tabulate: "IArray.list_of (IArray.tabulate (3, \i. i + 1))" +python_check ia_sub_pair: "IArray.sub' (IArray [10::int, 20], 1)" +python_check ia_sub_pair_var: "let p = (IArray [10::int, 20], 1) in IArray.sub' p" + +end diff --git a/test/oracle/Oracle_Import_Cycle.thy b/test/oracle/Oracle_Import_Cycle.thy new file mode 100644 index 0000000..3a34f7c --- /dev/null +++ b/test/oracle/Oracle_Import_Cycle.thy @@ -0,0 +1,12 @@ +theory Oracle_Import_Cycle + imports Python_Check "Python.Python_Setup" +begin + +text \Sorting string literals makes this module import \String\ before \Arith\. + The generated \Arith\ and \String\ import each other, and \Arith\ uses + \String.char_of_integer\ at import time. Kept apart because the failure is at + import, so it fails every check in its module.\ + +python_check sort_str: "sort [STR ''b'', STR ''a'', STR ''c'']" + +end diff --git a/test/oracle/Oracle_Keywords.thy b/test/oracle/Oracle_Keywords.thy new file mode 100644 index 0000000..6fd8586 --- /dev/null +++ b/test/oracle/Oracle_Keywords.thy @@ -0,0 +1,16 @@ +theory Oracle_Keywords + imports Python_Check "Python.Python_Setup" +begin + +text \Names that are Python keywords. Kept in their own module: if one is not + renamed, the module does not even parse.\ + +definition async :: int where "async = 1" +definition await :: int where "await = 2" +definition k_vars :: "int \ int \ int \ int" where + "k_vars nonlocal global del = nonlocal + global + del" + +python_check k_async: "async + await" +python_check k_vars: "k_vars 1 2 3" + +end diff --git a/test/oracle/Oracle_Names.thy b/test/oracle/Oracle_Names.thy new file mode 100644 index 0000000..ac16bdc --- /dev/null +++ b/test/oracle/Oracle_Names.thy @@ -0,0 +1,43 @@ +theory Oracle_Names + imports Python_Check "Python.Python_Setup" +begin + +text \Constants and variables named like Python builtins that the generated code + itself uses (\list\, \map\, \max\, \len\, \RuntimeError\, ...).\ + +definition v_list :: "int \ int list" where "v_list list = map (\x. x + list) [1, 2]" +definition v_len :: "int list \ nat" where "v_len len = length len" +definition v_ab :: "int \ int \ int" where "v_ab a b = b div a + a mod b" +definition v_capture :: "int \ int list" where "v_capture a = map (\b. b div a) [6, 7]" +definition v_self :: "int \ int \ int" where "v_self self cls = self - cls" +definition v_bool :: "bool \ int" where "v_bool bool = (if bool then 1 else 0)" +definition v_iter :: "int \ int \ int" where "v_iter iter next = iter + next" +definition v_print :: "int \ int" where "v_print print = print + 1" +definition v_str :: "int \ int" where "v_str str = str * 2" +definition v_type :: "int \ int" where "v_type type = type * 3" +definition v_kw :: "int \ int \ int \ int" where "v_kw pass yield lambda = pass + yield + lambda" + +definition RuntimeError :: "int \ int" where "RuntimeError x = x + 1" +definition NotImplementedError :: "int \ int" where "NotImplementedError x = x - 1" +definition object :: int where "object = 7" +definition tuple :: "int \ int \ int" where "tuple x = (x, x)" +definition dataclass :: int where "dataclass = 4" +definition Any :: int where "Any = 5" +definition functools :: int where "functools = 6" + +python_check n_list: "v_list 10" +python_check n_len: "v_len [1, 2, 3]" +python_check n_ab: "v_ab 3 7" +python_check n_capture: "v_capture 2" +python_check n_self: "v_self 5 3" +python_check n_bool: "v_bool True" +python_check n_iter: "v_iter 1 2" +python_check n_print: "v_print 1" +python_check n_str: "v_str 4" +python_check n_type: "v_type 4" +python_check n_kw: "v_kw 1 2 3" +python_check n_errors: "NotImplementedError (RuntimeError 5)" +python_check n_object_tuple: "tuple object" +python_check n_typing: "dataclass + Any + functools" + +end diff --git a/test/oracle/Oracle_Recursion.thy b/test/oracle/Oracle_Recursion.thy new file mode 100644 index 0000000..7be4952 --- /dev/null +++ b/test/oracle/Oracle_Recursion.thy @@ -0,0 +1,62 @@ +theory Oracle_Recursion + imports Python_Check "Python.Python_Setup" +begin + +text \Self tail calls whose new arguments depend on the old ones, which the + \python_tail_calls\ loop must rebind simultaneously.\ + +fun swap_count :: "nat \ int \ int \ int" where + "swap_count 0 a b = a - b" +| "swap_count (Suc n) a b = swap_count n b a" + +fun rot3 :: "nat \ int \ int \ int \ int" where + "rot3 0 a b c = 100 * a + 10 * b + c" +| "rot3 (Suc n) a b c = rot3 n c a b" + +fun dep_update :: "nat \ int \ int \ int" where + "dep_update 0 x y = x * 1000 + y" +| "dep_update (Suc n) x y = dep_update n (x + y) (x - y)" + +fun acc_rev :: "'a list \ 'a list \ 'a list" where + "acc_rev [] acc = acc" +| "acc_rev (x # xs) acc = acc_rev xs (x # acc)" + +fun count_pos :: "int list \ nat \ nat" where + "count_pos [] n = n" +| "count_pos (x # xs) n = (if x > 0 then count_pos xs (Suc n) else count_pos xs n)" + +fun my_gcd :: "nat \ nat \ nat" where + "my_gcd a b = (if b = 0 then a else my_gcd b (a mod b))" + +text \Closures created in the loop must not see later rebindings.\ + +fun add_all :: "int list \ (int \ int) \ int \ int" where + "add_all [] f x = f x" +| "add_all (y # ys) f x = add_all ys (\z. f z + y) x" + +fun ev :: "nat \ bool" and od :: "nat \ bool" where + "ev 0 = True" +| "ev (Suc n) = od n" +| "od 0 = False" +| "od (Suc n) = ev n" + +fun shift_left :: "nat \ 'a list \ 'a list" where + "shift_left n xs = (if n = 0 then xs else case xs of + [] \ [] | x # ys \ shift_left (n - 1) (ys @ [x]))" + +fun build :: "nat \ nat list \ nat list" where + "build 0 acc = acc" +| "build (Suc n) acc = build n (n # acc)" + +python_check_grid tc_build: "\n. build n []" over "[0, 1, 3]" +python_check_grid tc_swap: "swap_count" over "[0..<4]" "[10]" "[3]" +python_check_grid tc_rot: "rot3" over "[0, 1, 2, 4]" "[1]" "[2]" "[3]" +python_check_grid tc_dep: "dep_update" over "[0, 1, 2, 5]" "[3]" "[1]" +python_check tc_rev: "acc_rev [1::int, 2, 3] []" +python_check tc_count: "count_pos [1, -2, 3, 0, 5] 0" +python_check_grid tc_gcd: "my_gcd" over "[0, 12, 35]" "[0, 8, 14]" +python_check tc_closure: "add_all [1, 2, 3] (\z. z * 10) 1" +python_check_grid tc_mutual: "\n. (ev n, od n)" over "[0, 1, 6, 7]" +python_check_grid tc_shift: "\n. shift_left n [1::int, 2, 3]" over "[0, 1, 2, 4]" + +end diff --git a/test/oracle/Oracle_Strings_Bools.thy b/test/oracle/Oracle_Strings_Bools.thy new file mode 100644 index 0000000..6e75658 --- /dev/null +++ b/test/oracle/Oracle_Strings_Bools.thy @@ -0,0 +1,34 @@ +theory Oracle_Strings_Bools + imports Python_Check "Python.Python_Setup" +begin + +section \Strings and characters\ + +python_check str_concat: "STR ''ab'' + STR ''cd''" +python_check str_eq_chain: "(STR ''a'' = STR ''b'') = (STR ''c'' = STR ''d'')" +python_check str_eq_false: "(STR ''a'' = STR ''b'') = False" +python_check str_lt_chain: "(STR ''a'' < STR ''b'') = (STR ''c'' < STR ''b'')" +python_check str_le_eq: "(STR ''b'' \ STR ''a'') = (STR ''a'' = STR ''b'')" +python_check str_bool_eq: "(STR ''x'' = STR ''x'') = ((1::int) = 1)" +python_check str_compare: "(STR ''abc'' < STR ''abd'', STR ''b'' < STR ''abc'', STR '''' < STR ''a'')" +python_check str_concat_eq: "STR ''a'' + STR ''b'' = STR ''ab''" +python_check chr_eq: "(CHR ''a'' = CHR ''b'', CHR ''a'' = CHR ''a'')" +python_check chr_list: "''hello'' @ '' world''" +python_check chr_of_nat: "char_of (65::nat)" + +section \Booleans and precedence\ + +python_check b_not_eq: "(\ True) = False" +python_check b_int_eq_chain: "((1::int) = 2) = ((3::int) = 4)" +python_check b_bool_eq_chain: "(True = False) = (False = True)" +python_check b_imp: "(False \ False) \ \ (True \ False)" +python_check b_not_and: "\ (True \ False) \ (\ False \ False)" +python_check_grid b_imp_grid: "\a b. a \ b" over "[True, False]" "[True, False]" +python_check_grid b_eq_not: "\a b. (\ a) = b" over "[True, False]" "[True, False]" +python_check_grid b_eq_eq: "\a b c. (a = (b::bool)) = (c::bool)" over "[True, False]" "[True, False]" "[True, False]" +python_check_grid b_min_max: "\a b. (min a (b::bool), max a b, a < b, a \ b)" over "[True, False]" "[True, False]" +python_check b_if_arith: "(if True then 1 else 2::int) + (if False then 10 else 20)" +python_check b_neg_neg: "- (- (3::int))" +python_check b_if_cond_eq: "(if (1::int) = 1 = True then 1 else (0::int))" + +end diff --git a/test/oracle/Python_Check.thy b/test/oracle/Python_Check.thy new file mode 100644 index 0000000..d43b6d8 --- /dev/null +++ b/test/oracle/Python_Check.thy @@ -0,0 +1,195 @@ +theory Python_Check + imports Main + keywords "python_check" "python_check_grid" "python_export_checks" :: thy_decl + and "over" +begin + +text \ + Differential testing of the Python backend against Isabelle's SML evaluator. + + python_check name: "t" evaluates t to its value r with SML code evaluation and + defines two constants, both functions of a unit argument: + + name_ok computes t = r; just t if r is True, not t if r is False + (so a boolean term is not wrapped in an equality of its own) + name_actual computes t, for failure messages + + The generated Python must compute name_ok(None) as True. A negative control is + a check of t ~= r. + + python_check_grid name: "f" over "xs1" ... "xsn" adds one check per element of + the cartesian product of the argument lists. + + python_export_checks exports all checks of the theory to Python (like export_code + ... in Python file_prefix ".") together with a manifest oracle/manifest.json that + the pytest runner reads. +\ + +ML \ +structure Python_Check = +struct + +(*module: the theory that defines the check, i.e. the Python module that holds its constants*) +type check = {name: string, module: string, term: string, expected: string}; + +structure Data = Theory_Data +( + type T = check list; (*newest first*) + val empty = []; + fun merge (checks1, checks2) = Library.merge (op =) (checks1, checks2); +); + +fun const_names ({name, module, ...}: check) = + map (fn suffix => Long_Name.qualify module (name ^ suffix)) ["_ok", "_actual"]; + +fun string_of_term ctxt = XML.content_of o YXML.parse_body o Syntax.string_of_term ctxt; + +fun define name rhs lthy = + let + val b = Binding.name name; + in + lthy + |> Local_Theory.define ((b, NoSyn), ((Thm.def_binding b, @{attributes [code]}), rhs)) + |> snd + end; + +(*a function of unit, so that an exception only fails its own check, not the + import of the whole module*) +fun lazy t = Abs ("u", HOLogic.unitT, t); + +fun ok_term t r = + if r aconv \<^term>\True\ then t + else if r aconv \<^term>\False\ then HOLogic.mk_not t + else HOLogic.mk_eq (t, r); + +(*with Code_Abstract_Char, SML gives a character as Chr k, which a code equation + must not use; the literal is what the term would be written as anyway*) +fun char_literals (Const ("Code_Abstract_Char.Chr", _) $ k) = + HOLogic.mk_char (case k of + Const ("Code_Numeral.Pos", _) $ n => HOLogic.dest_numeral n + | _ => snd (HOLogic.dest_number k)) + | char_literals (t $ u) = char_literals t $ char_literals u + | char_literals (Abs (x, T, t)) = Abs (x, T, char_literals t) + | char_literals t = t; + +fun check name t lthy = + let + val thy = Proof_Context.theory_of lthy; + val module = Context.theory_base_name thy; + (*in a locale or class the constants would be named after it and take its + parameters, so neither const_names nor the runner's name_ok(None) would find them*) + val full_name = Local_Theory.full_name lthy (Binding.name (name ^ "_ok")); + val _ = full_name = Long_Name.qualify module (name ^ "_ok") orelse + error ("python_check " ^ quote name ^ ": not at theory level (would define " ^ + quote full_name ^ ")"); + val frees = Term.add_free_names t []; + val _ = null frees orelse + error ("python_check " ^ quote name ^ ": free variables " ^ commas_quote (rev frees)); + (*the constants would take an extra 'a itself argument, so name_ok(None) would + not compute the check*) + val tvars = Term.add_tfree_names t [] @ map (fst o fst) (Term.add_tvars t []); + val _ = null tvars orelse + error ("python_check " ^ quote name ^ ": type variables " ^ commas (rev tvars) ^ + ", add type annotations"); + val T = fastype_of t; + val _ = Sign.of_sort thy (T, \<^sort>\equal\) orelse + error ("python_check " ^ quote name ^ ": type has no equality: " ^ + Syntax.string_of_typ lthy T); + val r = char_literals (Code_Evaluation.dynamic_value_strict lthy t); + val check = {name = name, module = module, + term = string_of_term lthy t, expected = string_of_term lthy r}; + in + lthy + |> define (name ^ "_ok") (lazy (ok_term t r)) + |> define (name ^ "_actual") (lazy t) + |> Local_Theory.background_theory (Data.map (cons check)) + end; + +(*a list literal as written, so that its elements print as given, and any + other list, such as [0..<5], by its value*) +fun dest_arg_list lthy xs = + (case try HOLogic.dest_list xs of + SOME ys => ys + | NONE => HOLogic.dest_list (Code_Evaluation.dynamic_value_strict lthy xs)); + +fun check_grid name f xss lthy = + let + fun combos [] = [[]] + | combos (xs :: xss) = maps (fn x => map (cons x) (combos xss)) xs; + val argss = map (dest_arg_list lthy) xss; + val empty = map_filter (fn (i, xs) => if null xs then SOME (string_of_int (i + 1)) else NONE) + (map_index I argss); + val _ = + if null empty then () + else warning ("python_check_grid " ^ quote name ^ ": no checks, empty argument " ^ + (if length empty = 1 then "list " else "lists ") ^ commas empty); + val cases = combos argss; + in + fold_index (fn (i, xs) => + check (name ^ "_" ^ string_of_int i) (Envir.beta_norm (list_comb (f, xs)))) cases lthy + end; + +fun read_grid lthy (raw_f, raw_xss) = + let + val f = Syntax.read_term lthy raw_f; + val argTs = fst (strip_type (fastype_of f)); + val _ = length argTs >= length raw_xss orelse + error "python_check_grid: more argument lists than function arguments"; + val xss = map2 (fn T => fn raw => + Syntax.check_term lthy (Type.constraint (HOLogic.listT T) (Syntax.parse_term lthy raw))) + (take (length raw_xss) argTs) raw_xss; + in (f, xss) end; + +fun json_string s = + let + fun esc #"\"" = "\\\"" + | esc #"\\" = "\\\\" + | esc #"\n" = "\\n" + | esc c = + if Char.ord c < 32 + then "\\u" ^ StringCvt.padLeft #"0" 4 (Int.fmt StringCvt.HEX (Char.ord c)) + else String.str c; + in quote (String.translate esc s) end; + +fun json_of_check ({name, module, term, expected}: check) = + "{" ^ commas (map (fn (key, value) => json_string key ^ ": " ^ json_string value) + [("name", name), ("module", module), ("term", term), ("expected", expected)]) ^ "}"; + +fun export_checks lthy = + let + val thy = Proof_Context.theory_of lthy; + val checks = rev (Data.get thy); + val _ = null checks andalso error "python_export_checks: no checks"; + val manifest = + "{" ^ json_string "checks" ^ ": [\n " ^ + space_implode ",\n " (map json_of_check checks) ^ "\n]}\n"; + val _ = Export.export thy (Path.binding0 (Path.make ["oracle", "manifest.json"])) + [XML.Text manifest]; + in + Code_Target.export_code true (maps const_names checks) + [((("Python", ""), SOME ({physical = false}, (Path.current, Position.none))), [])] lthy + end; + +val _ = + Outer_Syntax.local_theory \<^command_keyword>\python_check\ + "check a term's Python value against SML evaluation" + (Parse.name -- (\<^keyword>\:\ |-- Parse.term) >> + (fn (name, raw_t) => fn lthy => check name (Syntax.read_term lthy raw_t) lthy)); + +val _ = + Outer_Syntax.local_theory \<^command_keyword>\python_check_grid\ + "check a function on all combinations of the given arguments" + (Parse.name -- (\<^keyword>\:\ |-- Parse.term) -- + (\<^keyword>\over\ |-- Scan.repeat1 Parse.term) >> + (fn ((name, raw_f), raw_xss) => fn lthy => + uncurry (check_grid name) (read_grid lthy (raw_f, raw_xss)) lthy)); + +val _ = + Outer_Syntax.local_theory \<^command_keyword>\python_export_checks\ + "export all checks of this theory to Python" + (Scan.succeed export_checks); + +end +\ + +end diff --git a/test/oracle/run_test.py b/test/oracle/run_test.py new file mode 100644 index 0000000..0dac17b --- /dev/null +++ b/test/oracle/run_test.py @@ -0,0 +1,109 @@ +""" +Differential tests: the Python generated for each python_check case in +test/oracle/Oracle_*.thy must agree with Isabelle's SML evaluation of the +same term. + +Prerequisite (run this first, from the project root): + + isabelle build -d . -e Python_Test_Oracle + +Then: + + uv run pytest test/oracle/run_test.py -q --tb=line + + +Each theory that runs python_export_checks exports oracle/manifest.json and +the generated modules under test/oracle/python_out/./. +Each check `c` in the manifest names the module that defines `c_ok` (the +generated `t = r`, where r is Isabelle's value of t) and `c_actual` (the +generated t, for failure messages), both functions of a unit argument. +""" + +import importlib +import inspect +import json +import sys +from pathlib import Path + +import pytest + +THIS_DIR = Path(__file__).resolve().parent +PYTHON_OUT_ROOT = THIS_DIR / "python_out" + +def _load_manifests(): + if not PYTHON_OUT_ROOT.exists(): + return [] + return sorted(PYTHON_OUT_ROOT.rglob("oracle/manifest.json")) + +def _cases(): + for manifest_path in _load_manifests(): + manifest = json.loads(manifest_path.read_text()) + code_dir = manifest_path.parent.parent / "code" + for check in manifest["checks"]: + yield pytest.param( + code_dir, check["module"], check, + id=f"{manifest_path.parent.parent.name}:{check['name']}", + ) + +_CASES = list(_cases()) +if not _CASES: + pytest.skip( + f"no oracle/manifest.json under {PYTHON_OUT_ROOT} -- run " + "`isabelle build -d . -e Python_Test_Oracle` first.", + allow_module_level=True, + ) + +_loaded = {} + +def _import(code_dir, module): + # Each export directory has its own copy of the same module names + # (Num, List, Oracle_Arith, ...), so forget the previous directory's + # modules before importing from another one. + key = (code_dir, module) + if key not in _loaded: + for path in code_dir.glob("*.py"): + sys.modules.pop(path.stem, None) + sys.path.insert(0, str(code_dir)) + try: + _loaded[key] = importlib.import_module(module) + finally: + sys.path.remove(str(code_dir)) + return _loaded[key] + + +@pytest.mark.parametrize("code_dir,module,check", _CASES) +def test_agrees_with_isabelle(code_dir, module, check): + # Each failure's first line names the term and the verdict, since that line is + # all that --tb=line shows. + term = check["term"] + try: + m = _import(code_dir, module) + except Exception as e: + pytest.fail(f"{term}: module {module} does not import: {_describe(e)}", + pytrace=False) + name = check["name"] + actual_fn = getattr(m, f"{name}_actual") + try: + ok = getattr(m, f"{name}_ok")(None) + except Exception as e: + pytest.fail(f"{term}: Isabelle gives {check['expected']}, " + f"Python raises {_describe(e)}\n {_source(actual_fn)}", + pytrace=False) + if ok is not True: + pytest.fail(f"{term}: Isabelle gives {check['expected']}, " + f"Python gives {actual_fn(None)!r}\n {_source(actual_fn)}", + pytrace=False) + + +def _describe(e): + message = str(e).splitlines()[0] if str(e) else "" + return f"{type(e).__name__}: {message}" if message else type(e).__name__ + + +def _source(fn): + code = fn.__code__ + try: + source = inspect.getsource(fn).strip() + except Exception: + source = "" + return f"{Path(code.co_filename).name}:{code.co_firstlineno}: {source}"