1 (* Title: HOL/Mutabelle/mutabelle_extra.ML
2 Author: Stefan Berghofer, Jasmin Blanchette, Lukas Bulwahn, TU Muenchen
4 Invokation of Counterexample generators.
7 signature MUTABELLE_EXTRA =
10 val take_random : int -> 'a list -> 'a list
12 datatype outcome = GenuineCex | PotentialCex | NoCex | Donno | Timeout | Error | Solved | Unsolved
13 type timings = (string * int) list
15 type mtd = string * (theory -> term -> outcome * timings)
17 type mutant_subentry = term * (string * (outcome * timings)) list
18 type detailed_entry = string * bool * term * mutant_subentry list
20 type subentry = string * int * int * int * int * int * int
21 type entry = string * bool * subentry list
22 type report = entry list
24 val quickcheck_mtd : (Proof.context -> Proof.context) -> string -> mtd
26 val solve_direct_mtd : mtd
29 val sledgehammer_mtd : mtd
37 val freezeT : term -> term
38 val thms_of : bool -> theory -> thm list
40 val string_for_report : report -> string
41 val write_report : string -> report -> unit
42 val mutate_theorems_and_write_report :
43 theory -> mtd list -> thm list -> string -> unit
45 val random_seed : real Unsynchronized.ref
48 structure MutabelleExtra : MUTABELLE_EXTRA =
51 (* Own seed; can't rely on the Isabelle one to stay the same *)
52 val random_seed = Unsynchronized.ref 1.0;
55 (* mutation options *)
57 val num_mutations = 1*)
58 (* soundness check: *)
62 (* quickcheck options *)
63 (*val quickcheck_generator = "SML"*)
65 (* Another Random engine *)
69 fun rmod x y = x - y * Real.realFloor (x / y);
76 fun random () = CRITICAL (fn () =>
77 let val r = rmod (a * ! random_seed) m
78 in (random_seed := r; r) end);
82 fun random_range l h =
83 if h < l orelse l < 0 then raise RANDOM
84 else l + Real.floor (rmod (random ()) (real (h - l + 1)));
86 fun take_random 0 _ = []
87 | take_random _ [] = []
89 let val j = random_range 0 (length xs - 1) in
90 Library.nth xs j :: take_random (n - 1) (nth_drop j xs)
93 (* possible outcomes *)
95 datatype outcome = GenuineCex | PotentialCex | NoCex | Donno | Timeout | Error | Solved | Unsolved
97 fun string_of_outcome GenuineCex = "GenuineCex"
98 | string_of_outcome PotentialCex = "PotentialCex"
99 | string_of_outcome NoCex = "NoCex"
100 | string_of_outcome Donno = "Donno"
101 | string_of_outcome Timeout = "Timeout"
102 | string_of_outcome Error = "Error"
103 | string_of_outcome Solved = "Solved"
104 | string_of_outcome Unsolved = "Unsolved"
106 type timings = (string * int) list
108 type mtd = string * (theory -> term -> outcome * timings)
110 type mutant_subentry = term * (string * (outcome * timings)) list
111 type detailed_entry = string * bool * term * mutant_subentry list
113 type subentry = string * int * int * int * int * int * int
114 type entry = string * bool * subentry list
115 type report = entry list
117 (* possible invocations *)
121 fun invoke_quickcheck change_options quickcheck_generator thy t =
122 TimeLimit.timeLimit (seconds (!Auto_Tools.time_limit))
125 val [result] = Quickcheck.test_goal_terms (change_options (Proof_Context.init_global thy))
126 (false, false) [] [(t, [])]
128 case Quickcheck.counterexample_of result of
129 NONE => (NoCex, Quickcheck.timings_of result)
130 | SOME _ => (GenuineCex, Quickcheck.timings_of result)
132 handle TimeLimit.TimeOut =>
133 (Timeout, [("timelimit", Real.floor (!Auto_Tools.time_limit))])
135 fun quickcheck_mtd change_options quickcheck_generator =
136 ("quickcheck_" ^ quickcheck_generator, invoke_quickcheck change_options quickcheck_generator)
140 fun invoke_solve_direct thy t =
142 val state = Proof.theorem NONE (K I) (map (single o rpair []) [t]) (Proof_Context.init_global thy)
144 case Solve_Direct.solve_direct false state of
145 (true, _) => (Solved, [])
146 | (false, _) => (Unsolved, [])
149 val solve_direct_mtd = ("solve_direct", invoke_solve_direct)
153 fun invoke_try thy t =
155 val state = Proof.theorem NONE (K I) (map (single o rpair []) [t]) (Proof_Context.init_global thy)
157 case Try.invoke_try (SOME (seconds 5.0)) ([], [], [], []) state of
159 | false => (Unsolved, [])
162 val try_mtd = ("try", invoke_try)
166 fun invoke_sledgehammer thy t =
167 if can (Goal.prove_global thy (Term.add_free_names t []) [] t)
168 (fn {context, ...} => Sledgehammer_Tactics.sledgehammer_with_metis_tac context 1) then
171 (Unsolved, ([], NONE))
173 val sledgehammer_mtd = ("sledgehammer", invoke_sledgehammer)
176 fun invoke_refute thy t =
178 val res = MyRefute.refute_term thy [] t
179 val _ = Output.urgent_message ("Refute: " ^ res)
182 "genuine" => GenuineCex
183 | "likely_genuine" => GenuineCex
184 | "potential" => PotentialCex
189 handle MyRefute.REFUTE (loc, details) =>
190 (error ("Unhandled Refute error (" ^ quote loc ^ "): " ^ details ^
192 val refute_mtd = ("refute", invoke_refute)
197 fun invoke_nitpick thy t =
199 val ctxt = Proof_Context.init_global thy
200 val state = Proof.init ctxt
201 val (res, _) = Nitpick.pick_nits_in_term state
202 (Nitpick_Isar.default_params thy []) false 1 1 1 [] [] t
203 val _ = Output.urgent_message ("Nitpick: " ^ res)
205 (rpair []) (case res of
206 "genuine" => GenuineCex
207 | "likely_genuine" => GenuineCex
208 | "potential" => PotentialCex
214 val nitpick_mtd = ("nitpick", invoke_nitpick)
216 (* filtering forbidden theorems and mutants *)
218 val comms = [@{const_name HOL.eq}, @{const_name HOL.disj}, @{const_name HOL.conj}]
221 [(* (@{const_name "power"}, "'a"), *)
222 (*(@{const_name induct_equal}, "'a"),
223 (@{const_name induct_implies}, "'a"),
224 (@{const_name induct_conj}, "'a"),*)
225 (@{const_name "undefined"}, "'a"),
226 (@{const_name "default"}, "'a"),
227 (@{const_name "dummy_pattern"}, "'a::{}"),
228 (@{const_name "HOL.simp_implies"}, "prop => prop => prop"),
229 (@{const_name "bot_fun_inst.bot_fun"}, "'a"),
230 (@{const_name "top_fun_inst.top_fun"}, "'a"),
231 (@{const_name "Pure.term"}, "'a"),
232 (@{const_name "top_class.top"}, "'a"),
233 (@{const_name "Quotient.Quot_True"}, "'a")(*,
234 (@{const_name "uminus"}, "'a"),
235 (@{const_name "Nat.size"}, "'a"),
236 (@{const_name "Groups.abs"}, "'a") *)]
239 ["finite_intvl_succ_class",
242 val forbidden_consts =
243 [@{const_name nibble_pair_of_char}, @{const_name "TYPE"}]
245 fun is_forbidden_theorem (s, th) =
246 let val consts = Term.add_const_names (prop_of th) [] in
247 exists (member (op =) (space_explode "." s)) forbidden_thms orelse
248 exists (member (op =) forbidden_consts) consts orelse
249 length (space_explode "." s) <> 2 orelse
250 String.isPrefix "type_definition" (List.last (space_explode "." s)) orelse
251 String.isSuffix "_def" s orelse
252 String.isSuffix "_raw" s orelse
253 String.isPrefix "term_of" (List.last (space_explode "." s))
256 val forbidden_mutant_constnames =
258 "HOL.induct_implies",
260 @{const_name undefined},
261 @{const_name default},
262 @{const_name dummy_pattern},
263 @{const_name "HOL.simp_implies"},
264 @{const_name "bot_fun_inst.bot_fun"},
265 @{const_name "top_fun_inst.top_fun"},
266 @{const_name "Pure.term"},
267 @{const_name "top_class.top"},
268 (*@{const_name "HOL.equal"},*)
269 @{const_name "Quotient.Quot_True"},
270 @{const_name "equal_fun_inst.equal_fun"},
271 @{const_name "equal_bool_inst.equal_bool"},
272 @{const_name "ord_fun_inst.less_eq_fun"},
273 @{const_name "ord_fun_inst.less_fun"},
274 @{const_name Metis.fequal},
275 @{const_name Meson.skolem},
276 @{const_name transfer_morphism}
277 (*@{const_name "==>"}, @{const_name "=="}*)]
279 val forbidden_mutant_consts =
281 (@{const_name "Groups.zero_class.zero"}, @{typ "prop => prop => prop"}),
282 (@{const_name "Groups.one_class.one"}, @{typ "prop => prop => prop"}),
283 (@{const_name "Groups.plus_class.plus"}, @{typ "prop => prop => prop"}),
284 (@{const_name "Groups.minus_class.minus"}, @{typ "prop => prop => prop"}),
285 (@{const_name "Groups.times_class.times"}, @{typ "prop => prop => prop"}),
286 (@{const_name "Rings.inverse_class.divide"}, @{typ "prop => prop => prop"}),
287 (@{const_name "Lattices.semilattice_inf_class.inf"}, @{typ "prop => prop => prop"}),
288 (@{const_name "Lattices.semilattice_sup_class.sup"}, @{typ "prop => prop => prop"}),
289 (@{const_name "Orderings.bot_class.bot"}, @{typ "prop => prop => prop"}),
290 (@{const_name "Orderings.ord_class.min"}, @{typ "prop => prop => prop"}),
291 (@{const_name "Orderings.ord_class.max"}, @{typ "prop => prop => prop"}),
292 (@{const_name "Divides.div_class.mod"}, @{typ "prop => prop => prop"}),
293 (@{const_name "Divides.div_class.div"}, @{typ "prop => prop => prop"}),
294 (@{const_name "GCD.gcd_class.gcd"}, @{typ "prop => prop => prop"}),
295 (@{const_name "GCD.gcd_class.lcm"}, @{typ "prop => prop => prop"}),
296 (@{const_name "Orderings.bot_class.bot"}, @{typ "bool => prop"}),
297 (@{const_name "Groups.one_class.one"}, @{typ "bool => prop"}),
298 (@{const_name "Groups.zero_class.zero"},@{typ "bool => prop"})]
300 fun is_forbidden_mutant t =
302 val const_names = Term.add_const_names t []
303 val consts = Term.add_consts t []
305 exists (String.isPrefix "Nitpick") const_names orelse
306 exists (String.isSubstring "_sumC") const_names orelse
307 exists (member (op =) forbidden_mutant_constnames) const_names orelse
308 exists (member (op =) forbidden_mutant_consts) consts
311 (* executable via quickcheck *)
313 fun is_executable_term thy t =
315 val ctxt = Proof_Context.init_global thy
317 can (TimeLimit.timeLimit (seconds 2.0)
318 (Quickcheck.test_goal_terms
319 ((Config.put Quickcheck.finite_types true #>
320 Config.put Quickcheck.finite_type_size 1 #>
321 Config.put Quickcheck.size 1 #> Config.put Quickcheck.iterations 1) ctxt)
322 (false, false) [])) (map (rpair [] o Object_Logic.atomize_term thy) (fst (Variable.import_terms true [t] ctxt)))
325 fun is_executable_thm thy th = is_executable_term thy (prop_of th)
328 map_types (map_type_tvar (fn ((a, i), S) =>
329 TFree (if i = 0 then a else a ^ "_" ^ string_of_int i, S)))
331 fun thms_of all thy =
333 (fn th => (all orelse Context.theory_name (theory_of_thm th) = Context.theory_name thy)
334 (* andalso is_executable_thm thy th *))
335 (map snd (filter_out is_forbidden_theorem (Mutabelle.all_unconcealed_thms_of thy)))
337 fun count x = (length oo filter o equal) x
339 fun cpu_time description e =
340 let val ({cpu, ...}, result) = Timing.timing e ()
341 in (result, (description, Time.toMilliseconds cpu)) end
343 fun unsafe_invoke_mtd thy (mtd_name, invoke_mtd) t =
345 val _ = Output.urgent_message ("Invoking " ^ mtd_name)
346 val ((res, (timing, reports)), time) = cpu_time "total time" (fn () => invoke_mtd thy t
347 handle ERROR s => (tracing s; (Error, ([], NONE))))
348 val _ = Output.urgent_message (" Done")
349 in (res, (time :: timing, reports)) end
351 fun safe_invoke_mtd thy (mtd_name, invoke_mtd) t =
353 val _ = Output.urgent_message ("Invoking " ^ mtd_name)
354 val (res, timing) = (*cpu_time "total time"
355 (fn () => *)case try (invoke_mtd thy) t of
356 SOME (res, timing) => (res, timing)
357 | NONE => (Output.urgent_message ("**** PROBLEMS WITH " ^ Syntax.string_of_term_global thy t);
359 val _ = Output.urgent_message (" Done")
362 (* theory -> term list -> mtd -> subentry *)
364 fun test_mutants_using_one_method thy mutants (mtd_name, invoke_mtd) =
366 val res = map (fst o safe_invoke_mtd thy (mtd_name, invoke_mtd)) mutants
368 (mtd_name, count GenuineCex res, count PotentialCex res, count NoCex res,
369 count Donno res, count Timeout res, count Error res)
372 (* creating entries *)
374 fun create_entry thy thm exec mutants mtds =
375 (Thm.get_name_hint thm, exec, map (test_mutants_using_one_method thy mutants) mtds)
377 fun create_detailed_entry thy thm exec mutants mtds =
379 fun create_mutant_subentry mutant = (mutant,
380 map (fn (mtd_name, invoke_mtd) =>
381 (mtd_name, safe_invoke_mtd thy (mtd_name, invoke_mtd) mutant)) mtds)
383 (Thm.get_name_hint thm, exec, prop_of thm, map create_mutant_subentry mutants)
386 (* (theory -> thm -> bool -> term list -> mtd list -> 'a) -> theory -> mtd list -> thm -> 'a *)
387 fun mutate_theorem create_entry thy mtds thm =
389 val exec = is_executable_thm thy thm
390 val _ = Output.tracing (if exec then "EXEC" else "NOEXEC")
392 (if num_mutations = 0 then
395 Mutabelle.mutate_mix (Thm.prop_of thm) thy comms forbidden
397 |> tap (fn muts => tracing ("mutants: " ^ string_of_int (length muts)))
398 |> filter_out is_forbidden_mutant
402 val _ = Output.urgent_message ("BEFORE PARTITION OF " ^
403 string_of_int (length mutants) ^ " MUTANTS")
404 val (execs, noexecs) = List.partition (is_executable_term thy) (take_random (20 * max_mutants) mutants)
405 val _ = tracing ("AFTER PARTITION (" ^ string_of_int (length execs) ^
406 " vs " ^ string_of_int (length noexecs) ^ ")")
408 execs @ take_random (Int.max (0, max_mutants - length execs)) noexecs
412 val mutants = mutants
413 |> map Mutabelle.freeze |> map freezeT
414 (* |> filter (not o is_forbidden_mutant) *)
415 |> map_filter (try (Sign.cert_term thy))
416 |> filter (is_some o try (Thm.cterm_of thy))
417 |> filter (is_some o try (Syntax.check_term (Proof_Context.init_global thy)))
418 |> take_random max_mutants
419 val _ = map (fn t => Output.urgent_message ("MUTANT: " ^ Syntax.string_of_term_global thy t)) mutants
421 create_entry thy thm exec mutants mtds
424 (* theory -> mtd list -> thm list -> report *)
425 val mutate_theorems = map ooo mutate_theorem
427 fun string_of_mutant_subentry thy thm_name (t, results) =
428 "mutant: " ^ Syntax.string_of_term_global thy t ^ "\n" ^
430 (map (fn (mtd_name, (outcome, timing)) => mtd_name ^ ": " ^ string_of_outcome outcome) results) ^
433 (* string -> string *)
434 val unyxml = XML.content_of o YXML.parse_body
436 fun string_of_mutant_subentry' thy thm_name (t, results) =
438 (* fun string_of_report (Quickcheck.Report {iterations = i, raised_match_errors = e,
439 satisfied_assms = s, positive_concl_tests = p}) =
440 "errors: " ^ string_of_int e ^ "; conclusion tests: " ^ string_of_int p
441 fun string_of_reports NONE = ""
442 | string_of_reports (SOME reports) =
443 cat_lines (map (fn (size, [report]) =>
444 "size " ^ string_of_int size ^ ": " ^ string_of_report report) (rev reports))*)
445 fun string_of_mtd_result (mtd_name, (outcome, timing)) =
446 mtd_name ^ ": " ^ string_of_outcome outcome
447 (*" with time " ^ " (" ^ space_implode "; " (map (fn (s, t) => (s ^ ": " ^ string_of_int t)) timing) ^ ")"*)
448 (*^ "\n" ^ string_of_reports reports*)
450 "mutant of " ^ thm_name ^ ":\n"
451 ^ unyxml (Syntax.string_of_term_global thy t) ^ "\n" ^ space_implode "; " (map string_of_mtd_result results)
454 fun string_of_detailed_entry thy (thm_name, exec, t, mutant_subentries) =
455 thm_name ^ " " ^ (if exec then "[exe]" else "[noexe]") ^ ": " ^
456 Syntax.string_of_term_global thy t ^ "\n" ^
457 cat_lines (map (string_of_mutant_subentry' thy thm_name) mutant_subentries) ^ "\n"
459 fun theoryfile_string_of_mutant_subentry thy thm_name (i, (t, results)) =
460 "lemma " ^ thm_name ^ "_" ^ string_of_int (i + 1) ^ ":\n" ^
461 "\"" ^ unyxml (Syntax.string_of_term_global thy t) ^
462 "\" \nquickcheck\noops\n"
464 fun theoryfile_string_of_detailed_entry thy (thm_name, exec, t, mutant_subentries) =
465 "subsubsection {* mutants of " ^ thm_name ^ " *}\n\n" ^
467 (theoryfile_string_of_mutant_subentry thy thm_name) mutant_subentries) ^ "\n"
469 (* subentry -> string *)
470 fun string_for_subentry (mtd_name, genuine_cex, potential_cex, no_cex, donno,
472 " " ^ mtd_name ^ ": " ^ string_of_int genuine_cex ^ "+ " ^
473 string_of_int potential_cex ^ "= " ^ string_of_int no_cex ^ "- " ^
474 string_of_int donno ^ "? " ^ string_of_int timeout ^ "T " ^
475 string_of_int error ^ "!"
477 (* entry -> string *)
478 fun string_for_entry (thm_name, exec, subentries) =
479 thm_name ^ " " ^ (if exec then "[exe]" else "[noexe]") ^ ":\n" ^
480 cat_lines (map string_for_subentry subentries) ^ "\n"
482 (* report -> string *)
483 fun string_for_report report = cat_lines (map string_for_entry report)
485 (* string -> report -> unit *)
486 fun write_report file_name =
487 File.write (Path.explode file_name) o string_for_report
489 (* theory -> mtd list -> thm list -> string -> unit *)
490 fun mutate_theorems_and_write_report thy mtds thms file_name =
492 val _ = Output.urgent_message "Starting Mutabelle..."
493 val ctxt = Proof_Context.init_global thy
494 val path = Path.explode file_name
495 (* for normal report: *)
497 val (gen_create_entry, gen_string_for_entry) = (create_entry, string_for_entry)
499 (* for detailled report: *)
500 val (gen_create_entry, gen_string_for_entry) = (create_detailed_entry, string_of_detailed_entry thy)
501 (* for theory creation: *)
502 (*val (gen_create_entry, gen_string_for_entry) = (create_detailed_entry, theoryfile_string_of_detailed_entry thy)*)
505 "Mutation options = " ^
506 "max_mutants: " ^ string_of_int max_mutants ^
507 "; num_mutations: " ^ string_of_int num_mutations ^ "\n" ^
509 (*"quickcheck_generator: " ^ quickcheck_generator ^ ";*)
510 "size: " ^ string_of_int (Config.get ctxt Quickcheck.size) ^
511 "; iterations: " ^ string_of_int (Config.get ctxt Quickcheck.iterations) ^ "\n");
512 map (File.append path o gen_string_for_entry o mutate_theorem gen_create_entry thy mtds) thms;