1 (* Title: HOL/Tools/Sledgehammer/sledgehammer_filter_mash.ML
2 Author: Jasmin Blanchette, TU Muenchen
4 Sledgehammer's machine-learning-based relevance filter (MaSh).
7 signature SLEDGEHAMMER_FILTER_MASH =
9 type status = ATP_Problem_Generate.status
10 type stature = ATP_Problem_Generate.stature
11 type fact = Sledgehammer_Fact.fact
12 type fact_override = Sledgehammer_Fact.fact_override
13 type params = Sledgehammer_Provers.params
14 type relevance_fudge = Sledgehammer_Provers.relevance_fudge
15 type prover_result = Sledgehammer_Provers.prover_result
17 val trace : bool Config.T
22 val fact_filters : string list
23 val escape_meta : string -> string
24 val escape_metas : string list -> string
25 val unescape_meta : string -> string
26 val unescape_metas : string -> string list
27 val extract_query : string -> string * string list
28 val suggested_facts : string list -> ('a * thm) list -> ('a * thm) list
30 int -> (('a * thm) list * ('a * thm) list) list -> ('a * thm) list
31 val is_likely_tautology_or_too_meta : thm -> bool
32 val theory_ord : theory * theory -> order
33 val thm_ord : thm * thm -> order
35 Proof.context -> string -> theory -> status -> term list -> string list
36 val isabelle_dependencies_of : unit Symtab.table -> thm -> string list
37 val goal_of_thm : theory -> thm -> thm
38 val run_prover_for_mash :
39 Proof.context -> params -> string -> fact list -> thm -> prover_result
40 val mash_RESET : Proof.context -> unit
43 -> (string * string list * string list * string list) list -> unit
46 -> (string * string list * string list * string list) list -> unit
48 Proof.context -> bool -> int -> string list * string list -> string list
49 val mash_reset : Proof.context -> unit
50 val mash_could_suggest_facts : unit -> bool
51 val mash_can_suggest_facts : Proof.context -> bool
52 val mash_suggest_facts :
53 Proof.context -> params -> string -> int -> term list -> term
54 -> ('a * thm) list -> ('a * thm) list * ('a * thm) list
56 Proof.context -> params -> theory -> Time.time -> fact list -> string
57 val mash_learn_proof :
58 Proof.context -> params -> term -> ('a * thm) list -> thm list -> unit
60 Proof.context -> params -> string -> int -> fact_override -> term list
61 -> term -> fact list -> fact list
62 val kill_learners : unit -> unit
63 val running_learners : unit -> unit
66 structure Sledgehammer_Filter_MaSh : SLEDGEHAMMER_FILTER_MASH =
70 open ATP_Problem_Generate
71 open Sledgehammer_Util
72 open Sledgehammer_Fact
73 open Sledgehammer_Filter_Iter
74 open Sledgehammer_Provers
75 open Sledgehammer_Minimize
78 Attrib.setup_config_bool @{binding sledgehammer_filter_mash_trace} (K false)
79 fun trace_msg ctxt msg = if Config.get ctxt trace then tracing (msg ()) else ()
87 val fact_filters = [meshN, iterN, mashN]
89 fun mash_home () = getenv "MASH_HOME"
90 fun mash_state_dir () =
91 getenv "ISABELLE_HOME_USER" ^ "/mash"
92 |> tap (Isabelle_System.mkdir o Path.explode)
93 fun mash_state_path () = mash_state_dir () ^ "/state" |> Path.explode
95 (*** Isabelle helpers ***)
98 if Char.isAlphaNum c orelse c = #"_" orelse c = #"." orelse c = #"(" orelse
99 c = #")" orelse c = #"," then
102 (* fixed width, in case more digits follow *)
103 "\\" ^ stringN_of_int 3 (Char.ord c)
105 fun unmeta_chars accum [] = String.implode (rev accum)
106 | unmeta_chars accum (#"\\" :: d1 :: d2 :: d3 :: cs) =
107 (case Int.fromString (String.implode [d1, d2, d3]) of
108 SOME n => unmeta_chars (Char.chr n :: accum) cs
109 | NONE => "" (* error *))
110 | unmeta_chars _ (#"\\" :: _) = "" (* error *)
111 | unmeta_chars accum (c :: cs) = unmeta_chars (c :: accum) cs
113 val escape_meta = String.translate meta_char
114 val escape_metas = map escape_meta #> space_implode " "
115 val unescape_meta = String.explode #> unmeta_chars []
117 space_explode " " #> filter_out (curry (op =) "") #> map unescape_meta
119 fun extract_query line =
120 case space_explode ":" line of
121 [goal_name, suggs] => (unescape_meta goal_name, unescape_metas suggs)
124 fun find_suggested facts sugg =
125 find_first (fn (_, th) => Thm.get_name_hint th = sugg) facts
126 fun suggested_facts suggs facts = map_filter (find_suggested facts) suggs
128 val scale_factor = 1000
130 fun scaled_powX x = Integer.pow 8 x
132 fun sum_sq_avg [] = 0
133 | sum_sq_avg xs = fold (Integer.add o scaled_powX) xs 0 div (length xs)
135 fun mesh_facts max_facts [(selected, unknown)] =
136 take max_facts selected @ take (max_facts - length selected) unknown
137 | mesh_facts max_facts mess =
139 val mess = mess |> map (apfst (`length))
140 val fact_eq = Thm.eq_thm o pairself snd
141 fun score_in fact ((sel_len, sels), unks) =
142 case find_index (curry fact_eq fact) sels of
143 ~1 => (case find_index (curry fact_eq fact) unks of
146 | j => SOME (scale_factor * (sel_len - j) div sel_len)
147 fun score_of fact = mess |> map_filter (score_in fact) |> sum_sq_avg
148 val facts = fold (union fact_eq o take max_facts o snd o fst) mess []
150 facts |> map (`score_of) |> sort (int_ord o swap o pairself fst)
151 |> tap (List.app (fn (score, (_, th)) => tracing ("score: " ^ string_of_int score ^ " " ^ Thm.get_name_hint th))
153 |> map snd |> take max_facts
156 val thy_feature_prefix = "y_"
158 val thy_feature_name_of = prefix thy_feature_prefix
159 val const_name_of = prefix const_prefix
160 val type_name_of = prefix type_const_prefix
161 val class_name_of = prefix class_prefix
163 fun is_likely_tautology_or_too_meta th =
165 val is_boring_const = member (op =) atp_widely_irrelevant_consts
166 fun is_boring_bool t =
167 not (exists_Const (not o is_boring_const o fst) t) orelse
168 exists_type (exists_subtype (curry (op =) @{typ prop})) t
169 fun is_boring_prop (@{const Trueprop} $ t) = is_boring_bool t
170 | is_boring_prop (@{const "==>"} $ t $ u) =
171 is_boring_prop t andalso is_boring_prop u
172 | is_boring_prop (Const (@{const_name all}, _) $ (Abs (_, _, t)) $ u) =
173 is_boring_prop t andalso is_boring_prop u
174 | is_boring_prop (Const (@{const_name "=="}, _) $ t $ u) =
175 is_boring_bool t andalso is_boring_bool u
176 | is_boring_prop _ = true
178 is_boring_prop (prop_of th) andalso not (Thm.eq_thm_prop (@{thm ext}, th))
182 if Theory.eq_thy p then
184 else if Theory.subthy p then
186 else if Theory.subthy (swap p) then
188 else case int_ord (pairself (length o Theory.ancestors_of) p) of
189 EQUAL => string_ord (pairself Context.theory_name p)
192 val thm_ord = theory_ord o pairself theory_of_thm
194 val bad_types = [@{type_name prop}, @{type_name bool}, @{type_name fun}]
196 fun interesting_terms_types_and_classes ctxt prover term_max_depth
199 fun is_bad_const (x as (s, _)) args =
200 member (op =) atp_logical_consts s orelse
201 fst (is_built_in_const_for_prover ctxt prover x args)
202 fun add_classes @{sort type} = I
203 | add_classes S = union (op =) (map class_name_of S)
204 fun do_add_type (Type (s, Ts)) =
205 (not (member (op =) bad_types s) ? insert (op =) (type_name_of s))
206 #> fold do_add_type Ts
207 | do_add_type (TFree (_, S)) = add_classes S
208 | do_add_type (TVar (_, S)) = add_classes S
209 fun add_type T = type_max_depth >= 0 ? do_add_type T
211 if member (op <>) args "" then s ^ "(" ^ space_implode "," args ^ ")"
213 fun patternify ~1 _ = ""
214 | patternify depth t =
216 (Const (s, _), args) =>
217 mk_app (const_name_of s) (map (patternify (depth - 1)) args)
219 fun add_term_patterns ~1 _ = I
220 | add_term_patterns depth t =
221 insert (op =) (patternify depth t)
222 #> add_term_patterns (depth - 1) t
223 val add_term = add_term_patterns term_max_depth
225 let val (head, args) = strip_comb t in
227 Const (x as (_, T)) =>
228 not (is_bad_const x args) ? (add_term t #> add_type T)
229 | Free (_, T) => add_type T
230 | Var (_, T) => add_type T
231 | Abs (_, T, body) => add_type T #> add_patterns body
233 #> fold add_patterns args
235 in [] |> fold add_patterns ts end
237 fun is_exists (s, _) = (s = @{const_name Ex} orelse s = @{const_name Ex1})
239 val term_max_depth = 1
240 val type_max_depth = 1
242 (* TODO: Generate type classes for types? *)
243 fun features_of ctxt prover thy status ts =
244 thy_feature_name_of (Context.theory_name thy) ::
245 interesting_terms_types_and_classes ctxt prover term_max_depth type_max_depth
247 |> exists (not o is_lambda_free) ts ? cons "lambdas"
248 |> exists (exists_Const is_exists) ts ? cons "skolems"
251 | Induction => cons "induction"
252 | Intro => cons "intro"
253 | Inductive => cons "inductive"
254 | Elim => cons "elim"
255 | Simp => cons "simp"
258 fun isabelle_dependencies_of all_facts = thms_in_proof (SOME all_facts)
260 val freezeT = Type.legacy_freeze_type
262 fun freeze (t $ u) = freeze t $ freeze u
263 | freeze (Abs (s, T, t)) = Abs (s, freezeT T, freeze t)
264 | freeze (Var ((s, _), T)) = Free (s, freezeT T)
265 | freeze (Const (s, T)) = Const (s, freezeT T)
266 | freeze (Free (s, T)) = Free (s, freezeT T)
269 fun goal_of_thm thy = prop_of #> freeze #> cterm_of thy #> Goal.init
271 fun run_prover_for_mash ctxt params prover facts goal =
274 {state = Proof.init ctxt, goal = goal, subgoal = 1, subgoal_count = 1,
275 facts = facts |> map (apfst (apfst (fn name => name ())))
276 |> map Untranslated_Fact}
277 val prover = get_minimizing_prover ctxt Normal (K ()) prover
278 in prover params (K (K (K ""))) problem end
281 (*** Low-level communication with MaSh ***)
283 fun write_file (xs, f) file =
284 let val path = Path.explode file in
287 |> List.app (File.append path o space_implode "" o map f)
290 fun mash_info overlord =
291 if overlord then (getenv "ISABELLE_HOME_USER", "")
292 else (getenv "ISABELLE_TMP", serial_string ())
294 fun run_mash ctxt (temp_dir, serial) core =
296 val log_file = temp_dir ^ "/mash_log" ^ serial
297 val err_file = temp_dir ^ "/mash_err" ^ serial
299 mash_home () ^ "/mash.py --quiet --outputDir " ^ mash_state_dir () ^
300 " --log " ^ log_file ^ " " ^ core ^ " 2>&1 > " ^ err_file
302 trace_msg ctxt (fn () => "Running " ^ command);
303 write_file ([], K "") log_file;
304 write_file ([], K "") err_file;
305 Isabelle_System.bash command; ()
308 fun run_mash_init ctxt overlord write_access write_feats write_deps =
310 val info as (temp_dir, serial) = mash_info overlord
311 val in_dir = temp_dir ^ "/mash_init" ^ serial
312 |> tap (Isabelle_System.mkdir o Path.explode)
314 write_file write_access (in_dir ^ "/mash_accessibility");
315 write_file write_feats (in_dir ^ "/mash_features");
316 write_file write_deps (in_dir ^ "/mash_dependencies");
317 run_mash ctxt info ("--init --inputDir " ^ in_dir)
320 fun run_mash_commands ctxt overlord save max_suggs write_cmds read_suggs =
322 val info as (temp_dir, serial) = mash_info overlord
323 val sugg_file = temp_dir ^ "/mash_suggs" ^ serial
324 val cmd_file = temp_dir ^ "/mash_commands" ^ serial
326 write_file ([], K "") sugg_file;
327 write_file write_cmds cmd_file;
329 ("--inputFile " ^ cmd_file ^ " --predictions " ^ sugg_file ^
330 " --numberOfPredictions " ^ string_of_int max_suggs ^
331 (if save then " --saveModel" else ""));
332 read_suggs (fn () => File.read_lines (Path.explode sugg_file))
335 fun str_of_update (name, parents, feats, deps) =
336 "! " ^ escape_meta name ^ ": " ^ escape_metas parents ^ "; " ^
337 escape_metas feats ^ "; " ^ escape_metas deps ^ "\n"
339 fun str_of_query (parents, feats) =
340 "? " ^ escape_metas parents ^ "; " ^ escape_metas feats
342 fun init_str_of_update get (upd as (name, _, _, _)) =
343 escape_meta name ^ ": " ^ escape_metas (get upd) ^ "\n"
345 fun mash_RESET ctxt =
346 let val path = mash_state_dir () |> Path.explode in
347 trace_msg ctxt (K "MaSh RESET");
348 File.fold_dir (fn file => fn () =>
349 File.rm (Path.append path (Path.basic file)))
353 fun mash_INIT ctxt _ [] = mash_RESET ctxt
354 | mash_INIT ctxt overlord upds =
355 (trace_msg ctxt (fn () => "MaSh INIT " ^
356 elide_string 1000 (space_implode " " (map #1 upds)));
357 run_mash_init ctxt overlord (upds, init_str_of_update #2)
358 (upds, init_str_of_update #3) (upds, init_str_of_update #4))
360 fun mash_ADD _ _ [] = ()
361 | mash_ADD ctxt overlord upds =
362 (trace_msg ctxt (fn () => "MaSh ADD " ^
363 elide_string 1000 (space_implode " " (map #1 upds)));
364 run_mash_commands ctxt overlord true 0 (upds, str_of_update) (K ()))
366 fun mash_QUERY ctxt overlord max_suggs (query as (_, feats)) =
367 (trace_msg ctxt (fn () => "MaSh QUERY " ^ space_implode " " feats);
368 run_mash_commands ctxt overlord false max_suggs
369 ([query], str_of_query)
370 (fn suggs => snd (extract_query (List.last (suggs ()))))
371 handle List.Empty => [])
374 (*** High-level communication with MaSh ***)
376 fun try_graph ctxt when def f =
378 handle Graph.CYCLES (cycle :: _) =>
379 (trace_msg ctxt (fn () =>
380 "Cycle involving " ^ commas cycle ^ " when " ^ when); def)
381 | Graph.UNDEF name =>
382 (trace_msg ctxt (fn () =>
383 "Unknown fact " ^ quote name ^ " when " ^ when); def)
386 {thys : bool Symtab.table,
387 fact_graph : unit Graph.T}
389 val empty_state = {thys = Symtab.empty, fact_graph = Graph.empty}
393 fun mash_load _ (state as (true, _)) = state
395 let val path = mash_state_path () in
397 case try File.read_lines path of
398 SOME (comp_thys :: incomp_thys :: fact_lines) =>
400 fun add_thy comp thy = Symtab.update (thy, comp)
401 fun add_edge_to name parent =
402 Graph.default_node (parent, ())
403 #> Graph.add_edge (parent, name)
404 fun add_fact_line line =
405 case extract_query line of
406 ("", _) => I (* shouldn't happen *)
408 Graph.default_node (name, ())
409 #> fold (add_edge_to name) parents
411 Symtab.empty |> fold (add_thy true) (unescape_metas comp_thys)
412 |> fold (add_thy false) (unescape_metas incomp_thys)
414 try_graph ctxt "loading state" Graph.empty (fn () =>
415 Graph.empty |> fold add_fact_line fact_lines)
416 in {thys = thys, fact_graph = fact_graph} end
420 fun mash_save ({thys, fact_graph, ...} : mash_state) =
422 val path = mash_state_path ()
423 val thys = Symtab.dest thys
424 val line_for_thys = escape_metas o AList.find (op =) thys
425 fun fact_line_for name parents =
426 escape_meta name ^ ": " ^ escape_metas parents
427 val append_fact = File.append path o suffix "\n" oo fact_line_for
429 File.write path (line_for_thys true ^ "\n" ^ line_for_thys false ^ "\n");
430 Graph.fold (fn (name, ((), (parents, _))) => fn () =>
431 append_fact name (Graph.Keys.dest parents))
436 Synchronized.var "Sledgehammer_Filter_MaSh.global_state" (false, empty_state)
440 fun mash_map ctxt f =
441 Synchronized.change global_state (mash_load ctxt ##> (f #> tap mash_save))
444 Synchronized.change_result global_state (mash_load ctxt #> `snd)
446 fun mash_reset ctxt =
447 Synchronized.change global_state (fn _ =>
448 (mash_RESET ctxt; File.write (mash_state_path ()) "";
449 (true, empty_state)))
453 fun mash_could_suggest_facts () = mash_home () <> ""
454 fun mash_can_suggest_facts ctxt =
455 not (Graph.is_empty (#fact_graph (mash_get ctxt)))
457 fun parents_wrt_facts ctxt facts fact_graph =
459 val graph_facts = Symtab.make (map (rpair ()) (Graph.keys fact_graph))
461 try_graph ctxt "when computing ancestor facts" [] (fn () =>
462 [] |> fold (cons o Thm.get_name_hint o snd) facts
463 |> filter (Symtab.defined graph_facts)
464 |> Graph.all_preds fact_graph)
465 val facts = Symtab.empty |> fold (fn name => Symtab.update (name, ())) facts
467 try_graph ctxt "when computing parent facts" [] (fn () =>
468 fact_graph |> Graph.restrict (Symtab.defined facts) |> Graph.maximals)
471 (* Generate more suggestions than requested, because some might be thrown out
472 later for various reasons and "meshing" gives better results with some
474 fun max_suggs_of max_facts = max_facts + Int.min (200, max_facts)
476 fun is_fact_in_graph fact_graph (_, th) =
477 can (Graph.get_node fact_graph) (Thm.get_name_hint th)
479 fun mash_suggest_facts ctxt ({overlord, ...} : params) prover max_facts hyp_ts
482 val thy = Proof_Context.theory_of ctxt
483 val fact_graph = #fact_graph (mash_get ctxt)
484 val parents = parents_wrt_facts ctxt facts fact_graph
485 val feats = features_of ctxt prover thy General (concl_t :: hyp_ts)
487 if Graph.is_empty fact_graph then []
488 else mash_QUERY ctxt overlord (max_suggs_of max_facts) (parents, feats)
489 val selected = facts |> suggested_facts suggs
490 val unknown = facts |> filter_out (is_fact_in_graph fact_graph)
491 in (selected, unknown) end
493 fun add_thys_for thy =
494 let fun add comp thy = Symtab.update (Context.theory_name thy, comp) in
495 add false thy #> fold (add true) (Theory.ancestors_of thy)
498 fun update_fact_graph ctxt (name, parents, feats, deps) (upds, graph) =
500 fun maybe_add_from from (accum as (parents, graph)) =
501 try_graph ctxt "updating graph" accum (fn () =>
502 (from :: parents, Graph.add_edge_acyclic (from, name) graph))
503 val graph = graph |> Graph.default_node (name, ())
504 val (parents, graph) = ([], graph) |> fold maybe_add_from parents
505 val (deps, graph) = ([], graph) |> fold maybe_add_from deps
506 in ((name, parents, feats, deps) :: upds, graph) end
508 val pass1_learn_timeout_factor = 0.5
510 (* The timeout is understood in a very slack fashion. *)
511 fun mash_learn_thy ctxt ({provers, verbose, overlord, ...} : params) thy timeout
514 val timer = Timer.startRealTimer ()
515 val prover = hd provers
517 Time.> (Timer.checkRealTimer timer, time_mult frac timeout)
518 val {fact_graph, ...} = mash_get ctxt
520 facts |> filter_out (is_fact_in_graph fact_graph)
521 |> sort (thm_ord o pairself snd)
523 if null new_facts then
527 val ths = facts |> map snd
529 ths |> filter_out is_likely_tautology_or_too_meta
530 |> map (rpair () o Thm.get_name_hint)
532 fun do_fact _ (accum as (_, true)) = accum
533 | do_fact ((_, (_, status)), th) ((parents, upds), false) =
535 val name = Thm.get_name_hint th
536 val feats = features_of ctxt prover thy status [prop_of th]
537 val deps = isabelle_dependencies_of all_names th
538 val upd = (name, parents, feats, deps)
539 in (([name], upd :: upds), timed_out pass1_learn_timeout_factor) end
540 val parents = parents_wrt_facts ctxt facts fact_graph
542 ((parents, []), false) |> fold do_fact new_facts |>> apsnd rev
544 fun trans {thys, fact_graph} =
546 val mash_INIT_or_ADD =
547 if Graph.is_empty fact_graph then mash_INIT else mash_ADD
548 val (upds, fact_graph) =
549 ([], fact_graph) |> fold (update_fact_graph ctxt) upds
551 (mash_INIT_or_ADD ctxt overlord (rev upds);
552 {thys = thys |> add_thys_for thy,
553 fact_graph = fact_graph})
558 "Processed " ^ string_of_int n ^ " proof" ^ plural_s n ^
560 " in " ^ string_from_time (Timer.checkRealTimer timer)
568 fun mash_learn_proof ctxt ({provers, overlord, ...} : params) t facts used_ths =
570 val thy = Proof_Context.theory_of ctxt
571 val prover = hd provers
572 val name = ATP_Util.timestamp () ^ " " ^ serial_string () (* fresh enough *)
573 val feats = features_of ctxt prover thy General [t]
574 val deps = used_ths |> map Thm.get_name_hint
576 mash_map ctxt (fn {thys, fact_graph} =>
578 val parents = parents_wrt_facts ctxt facts fact_graph
579 val upds = [(name, parents, feats, deps)]
580 val (upds, fact_graph) =
581 ([], fact_graph) |> fold (update_fact_graph ctxt) upds
583 mash_ADD ctxt overlord upds;
584 {thys = thys, fact_graph = fact_graph}
588 (* The threshold should be large enough so that MaSh doesn't kick in for Auto
589 Sledgehammer and Try. *)
590 val min_secs_for_learning = 15
591 val learn_timeout_factor = 2.0
593 fun relevant_facts ctxt (params as {learn, fact_filter, timeout, ...}) prover
594 max_facts ({add, only, ...} : fact_override) hyp_ts concl_t facts =
595 if not (subset (op =) (the_list fact_filter, fact_filters)) then
596 error ("Unknown fact filter: " ^ quote (the fact_filter) ^ ".")
599 else if max_facts <= 0 orelse null facts then
603 val thy = Proof_Context.theory_of ctxt
605 if not learn orelse Async_Manager.has_running_threads MaShN then
607 else if Time.toSeconds timeout >= min_secs_for_learning then
609 val soft_timeout = time_mult learn_timeout_factor timeout
610 val hard_timeout = time_mult 4.0 soft_timeout
611 val birth_time = Time.now ()
612 val death_time = Time.+ (birth_time, hard_timeout)
613 val desc = ("machine learner for Sledgehammer", "")
615 Async_Manager.launch MaShN birth_time death_time desc
617 (true, mash_learn_thy ctxt params thy soft_timeout facts))
623 SOME ff => (() |> ff <> iterN ? maybe_learn; ff)
625 if mash_can_suggest_facts ctxt then (maybe_learn (); meshN)
626 else if mash_could_suggest_facts () then (maybe_learn (); iterN)
628 val add_ths = Attrib.eval_thms ctxt add
629 fun prepend_facts ths accepts =
630 ((facts |> filter (member Thm.eq_thm_prop ths o snd)) @
631 (accepts |> filter_out (member Thm.eq_thm_prop ths o snd)))
634 iterative_relevant_facts ctxt params prover max_facts NONE hyp_ts
637 mash_suggest_facts ctxt params prover max_facts hyp_ts concl_t facts
639 [] |> (if fact_filter <> mashN then cons (iter (), []) else I)
640 |> (if fact_filter <> iterN then cons (mash ()) else I)
642 mesh_facts max_facts mess
643 |> not (null add_ths) ? prepend_facts add_ths
646 fun kill_learners () = Async_Manager.kill_threads MaShN "learner"
647 fun running_learners () = Async_Manager.running_threads MaShN "learner"