1 (* Title: HOL/Tools/Sledgehammer/sledgehammer_filter_mash.ML
2 Author: Jasmin Blanchette, TU Muenchen
4 Sledgehammer's machine-learning-based relevance filter (MaSh).
7 signature SLEDGEHAMMER_FILTER_MASH =
9 type status = ATP_Problem_Generate.status
10 type stature = ATP_Problem_Generate.stature
11 type fact = Sledgehammer_Fact.fact
12 type fact_override = Sledgehammer_Fact.fact_override
13 type params = Sledgehammer_Provers.params
14 type relevance_fudge = Sledgehammer_Provers.relevance_fudge
15 type prover_result = Sledgehammer_Provers.prover_result
17 val trace : bool Config.T
22 val fact_filters : string list
23 val escape_meta : string -> string
24 val escape_metas : string list -> string
25 val unescape_meta : string -> string
26 val unescape_metas : string -> string list
27 val extract_query : string -> string * string list
28 val suggested_facts : string list -> ('a * thm) list -> ('a * thm) list
30 int -> (('a * thm) list * ('a * thm) list) list -> ('a * thm) list
31 val is_likely_tautology_or_too_meta : thm -> bool
32 val theory_ord : theory * theory -> order
33 val thm_ord : thm * thm -> order
35 Proof.context -> string -> theory -> status -> term list -> string list
36 val isabelle_dependencies_of : unit Symtab.table -> thm -> string list
37 val goal_of_thm : theory -> thm -> thm
38 val run_prover_for_mash :
39 Proof.context -> params -> string -> fact list -> thm -> prover_result
40 val mash_RESET : Proof.context -> unit
43 -> (string * string list * string list * string list) list -> unit
46 -> (string * string list * string list * string list) list -> unit
48 Proof.context -> bool -> int -> string list * string list -> string list
49 val mash_reset : Proof.context -> unit
50 val mash_could_suggest_facts : unit -> bool
51 val mash_can_suggest_facts : Proof.context -> bool
52 val mash_suggest_facts :
53 Proof.context -> params -> string -> int -> term list -> term
54 -> ('a * thm) list -> ('a * thm) list * ('a * thm) list
56 Proof.context -> params -> theory -> Time.time -> fact list -> string
57 val mash_learn_proof :
58 Proof.context -> params -> term -> ('a * thm) list -> thm list -> unit
60 Proof.context -> params -> string -> int -> fact_override -> term list
61 -> term -> fact list -> fact list
62 val kill_learners : unit -> unit
63 val running_learners : unit -> unit
66 structure Sledgehammer_Filter_MaSh : SLEDGEHAMMER_FILTER_MASH =
70 open ATP_Problem_Generate
71 open Sledgehammer_Util
72 open Sledgehammer_Fact
73 open Sledgehammer_Filter_Iter
74 open Sledgehammer_Provers
75 open Sledgehammer_Minimize
78 Attrib.setup_config_bool @{binding sledgehammer_filter_mash_trace} (K false)
79 fun trace_msg ctxt msg = if Config.get ctxt trace then tracing (msg ()) else ()
87 val fact_filters = [meshN, iterN, mashN]
89 fun mash_home () = getenv "MASH_HOME"
90 fun mash_state_dir () =
91 getenv "ISABELLE_HOME_USER" ^ "/mash"
92 |> tap (Isabelle_System.mkdir o Path.explode)
93 fun mash_state_path () = mash_state_dir () ^ "/state" |> Path.explode
95 (*** Isabelle helpers ***)
98 if Char.isAlphaNum c orelse c = #"_" orelse c = #"." orelse c = #"(" orelse
99 c = #")" orelse c = #"," then
102 (* fixed width, in case more digits follow *)
103 "\\" ^ stringN_of_int 3 (Char.ord c)
105 fun unmeta_chars accum [] = String.implode (rev accum)
106 | unmeta_chars accum (#"\\" :: d1 :: d2 :: d3 :: cs) =
107 (case Int.fromString (String.implode [d1, d2, d3]) of
108 SOME n => unmeta_chars (Char.chr n :: accum) cs
109 | NONE => "" (* error *))
110 | unmeta_chars _ (#"\\" :: _) = "" (* error *)
111 | unmeta_chars accum (c :: cs) = unmeta_chars (c :: accum) cs
113 val escape_meta = String.translate meta_char
114 val escape_metas = map escape_meta #> space_implode " "
115 val unescape_meta = String.explode #> unmeta_chars []
117 space_explode " " #> filter_out (curry (op =) "") #> map unescape_meta
119 fun extract_query line =
120 case space_explode ":" line of
121 [goal_name, suggs] => (unescape_meta goal_name, unescape_metas suggs)
124 fun find_suggested facts sugg =
125 find_first (fn (_, th) => Thm.get_name_hint th = sugg) facts
126 fun suggested_facts suggs facts = map_filter (find_suggested facts) suggs
128 fun sum_avg _ [] = 1000000000 (* big number *)
129 | sum_avg n xs = fold (Integer.add o Integer.mult n) xs 0 div (length xs)
131 fun mesh_facts max_facts [(selected, unknown)] =
132 take max_facts selected @ take (max_facts - length selected) unknown
133 | mesh_facts max_facts mess =
135 val mess = mess |> map (apfst (`length))
137 val fact_eq = Thm.eq_thm o pairself snd
138 fun score_in fact ((sel_len, sels), unks) =
139 case find_index (curry fact_eq fact) sels of
140 ~1 => (case find_index (curry fact_eq fact) unks of
144 fun score_of fact = mess |> map_filter (score_in fact) |> sum_avg n
145 val facts = fold (union fact_eq o take max_facts o snd o fst) mess []
147 facts |> map (`score_of) |> sort (int_ord o pairself fst) |> map snd
151 val thy_feature_prefix = "y_"
153 val thy_feature_name_of = prefix thy_feature_prefix
154 val const_name_of = prefix const_prefix
155 val type_name_of = prefix type_const_prefix
156 val class_name_of = prefix class_prefix
158 fun is_likely_tautology_or_too_meta th =
160 val is_boring_const = member (op =) atp_widely_irrelevant_consts
161 fun is_boring_bool t =
162 not (exists_Const (not o is_boring_const o fst) t) orelse
163 exists_type (exists_subtype (curry (op =) @{typ prop})) t
164 fun is_boring_prop (@{const Trueprop} $ t) = is_boring_bool t
165 | is_boring_prop (@{const "==>"} $ t $ u) =
166 is_boring_prop t andalso is_boring_prop u
167 | is_boring_prop (Const (@{const_name all}, _) $ (Abs (_, _, t)) $ u) =
168 is_boring_prop t andalso is_boring_prop u
169 | is_boring_prop (Const (@{const_name "=="}, _) $ t $ u) =
170 is_boring_bool t andalso is_boring_bool u
171 | is_boring_prop _ = true
173 is_boring_prop (prop_of th) andalso not (Thm.eq_thm_prop (@{thm ext}, th))
177 if Theory.eq_thy p then
179 else if Theory.subthy p then
181 else if Theory.subthy (swap p) then
183 else case int_ord (pairself (length o Theory.ancestors_of) p) of
184 EQUAL => string_ord (pairself Context.theory_name p)
187 val thm_ord = theory_ord o pairself theory_of_thm
189 val bad_types = [@{type_name prop}, @{type_name bool}, @{type_name fun}]
191 fun interesting_terms_types_and_classes ctxt prover term_max_depth
194 fun is_bad_const (x as (s, _)) args =
195 member (op =) atp_logical_consts s orelse
196 fst (is_built_in_const_for_prover ctxt prover x args)
197 fun add_classes @{sort type} = I
198 | add_classes S = union (op =) (map class_name_of S)
199 fun do_add_type (Type (s, Ts)) =
200 (not (member (op =) bad_types s) ? insert (op =) (type_name_of s))
201 #> fold do_add_type Ts
202 | do_add_type (TFree (_, S)) = add_classes S
203 | do_add_type (TVar (_, S)) = add_classes S
204 fun add_type T = type_max_depth >= 0 ? do_add_type T
206 if member (op <>) args "" then s ^ "(" ^ space_implode "," args ^ ")"
208 fun patternify ~1 _ = ""
209 | patternify depth t =
211 (Const (s, _), args) =>
212 mk_app (const_name_of s) (map (patternify (depth - 1)) args)
214 fun add_term_patterns ~1 _ = I
215 | add_term_patterns depth t =
216 insert (op =) (patternify depth t)
217 #> add_term_patterns (depth - 1) t
218 val add_term = add_term_patterns term_max_depth
220 let val (head, args) = strip_comb t in
222 Const (x as (_, T)) =>
223 not (is_bad_const x args) ? (add_term t #> add_type T)
224 | Free (_, T) => add_type T
225 | Var (_, T) => add_type T
226 | Abs (_, T, body) => add_type T #> add_patterns body
228 #> fold add_patterns args
230 in [] |> fold add_patterns ts end
232 fun is_exists (s, _) = (s = @{const_name Ex} orelse s = @{const_name Ex1})
234 val term_max_depth = 1
235 val type_max_depth = 1
237 (* TODO: Generate type classes for types? *)
238 fun features_of ctxt prover thy status ts =
239 thy_feature_name_of (Context.theory_name thy) ::
240 interesting_terms_types_and_classes ctxt prover term_max_depth type_max_depth
242 |> exists (not o is_lambda_free) ts ? cons "lambdas"
243 |> exists (exists_Const is_exists) ts ? cons "skolems"
246 | Induction => cons "induction"
247 | Intro => cons "intro"
248 | Inductive => cons "inductive"
249 | Elim => cons "elim"
250 | Simp => cons "simp"
253 fun isabelle_dependencies_of all_facts = thms_in_proof (SOME all_facts)
255 val freezeT = Type.legacy_freeze_type
257 fun freeze (t $ u) = freeze t $ freeze u
258 | freeze (Abs (s, T, t)) = Abs (s, freezeT T, freeze t)
259 | freeze (Var ((s, _), T)) = Free (s, freezeT T)
260 | freeze (Const (s, T)) = Const (s, freezeT T)
261 | freeze (Free (s, T)) = Free (s, freezeT T)
264 fun goal_of_thm thy = prop_of #> freeze #> cterm_of thy #> Goal.init
266 fun run_prover_for_mash ctxt params prover facts goal =
269 {state = Proof.init ctxt, goal = goal, subgoal = 1, subgoal_count = 1,
270 facts = facts |> map (apfst (apfst (fn name => name ())))
271 |> map Untranslated_Fact}
272 val prover = get_minimizing_prover ctxt Normal (K ()) prover
273 in prover params (K (K (K ""))) problem end
276 (*** Low-level communication with MaSh ***)
278 fun write_file (xs, f) file =
279 let val path = Path.explode file in
282 |> List.app (File.append path o space_implode "" o map f)
285 fun mash_info overlord =
286 if overlord then (getenv "ISABELLE_HOME_USER", "")
287 else (getenv "ISABELLE_TMP", serial_string ())
289 fun run_mash ctxt (temp_dir, serial) core =
291 val log_file = temp_dir ^ "/mash_log" ^ serial
292 val err_file = temp_dir ^ "/mash_err" ^ serial
294 mash_home () ^ "/mash.py --quiet --outputDir " ^ mash_state_dir () ^
295 " --log " ^ log_file ^ " " ^ core ^ " 2>&1 > " ^ err_file
297 trace_msg ctxt (fn () => "Running " ^ command);
298 write_file ([], K "") log_file;
299 write_file ([], K "") err_file;
300 Isabelle_System.bash command; ()
303 fun run_mash_init ctxt overlord write_access write_feats write_deps =
305 val info as (temp_dir, serial) = mash_info overlord
306 val in_dir = temp_dir ^ "/mash_init" ^ serial
307 |> tap (Isabelle_System.mkdir o Path.explode)
309 write_file write_access (in_dir ^ "/mash_accessibility");
310 write_file write_feats (in_dir ^ "/mash_features");
311 write_file write_deps (in_dir ^ "/mash_dependencies");
312 run_mash ctxt info ("--init --inputDir " ^ in_dir)
315 fun run_mash_commands ctxt overlord save max_suggs write_cmds read_suggs =
317 val info as (temp_dir, serial) = mash_info overlord
318 val sugg_file = temp_dir ^ "/mash_suggs" ^ serial
319 val cmd_file = temp_dir ^ "/mash_commands" ^ serial
321 write_file ([], K "") sugg_file;
322 write_file write_cmds cmd_file;
324 ("--inputFile " ^ cmd_file ^ " --predictions " ^ sugg_file ^
325 " --numberOfPredictions " ^ string_of_int max_suggs ^
326 (if save then " --saveModel" else ""));
327 read_suggs (fn () => File.read_lines (Path.explode sugg_file))
330 fun str_of_update (name, parents, feats, deps) =
331 "! " ^ escape_meta name ^ ": " ^ escape_metas parents ^ "; " ^
332 escape_metas feats ^ "; " ^ escape_metas deps ^ "\n"
334 fun str_of_query (parents, feats) =
335 "? " ^ escape_metas parents ^ "; " ^ escape_metas feats
337 fun init_str_of_update get (upd as (name, _, _, _)) =
338 escape_meta name ^ ": " ^ escape_metas (get upd) ^ "\n"
340 fun mash_RESET ctxt =
341 let val path = mash_state_dir () |> Path.explode in
342 trace_msg ctxt (K "MaSh RESET");
343 File.fold_dir (fn file => fn () =>
344 File.rm (Path.append path (Path.basic file)))
348 fun mash_INIT ctxt _ [] = mash_RESET ctxt
349 | mash_INIT ctxt overlord upds =
350 (trace_msg ctxt (fn () => "MaSh INIT " ^
351 elide_string 1000 (space_implode " " (map #1 upds)));
352 run_mash_init ctxt overlord (upds, init_str_of_update #2)
353 (upds, init_str_of_update #3) (upds, init_str_of_update #4))
355 fun mash_ADD _ _ [] = ()
356 | mash_ADD ctxt overlord upds =
357 (trace_msg ctxt (fn () => "MaSh ADD " ^
358 elide_string 1000 (space_implode " " (map #1 upds)));
359 run_mash_commands ctxt overlord true 0 (upds, str_of_update) (K ()))
361 fun mash_QUERY ctxt overlord max_suggs (query as (_, feats)) =
362 (trace_msg ctxt (fn () => "MaSh QUERY " ^ space_implode " " feats);
363 run_mash_commands ctxt overlord false max_suggs
364 ([query], str_of_query)
365 (fn suggs => snd (extract_query (List.last (suggs ()))))
366 handle List.Empty => [])
369 (*** High-level communication with MaSh ***)
371 fun try_graph ctxt when def f =
373 handle Graph.CYCLES (cycle :: _) =>
374 (trace_msg ctxt (fn () =>
375 "Cycle involving " ^ commas cycle ^ " when " ^ when); def)
376 | Graph.UNDEF name =>
377 (trace_msg ctxt (fn () =>
378 "Unknown fact " ^ quote name ^ " when " ^ when); def)
381 {thys : bool Symtab.table,
382 fact_graph : unit Graph.T}
384 val empty_state = {thys = Symtab.empty, fact_graph = Graph.empty}
388 fun mash_load _ (state as (true, _)) = state
390 let val path = mash_state_path () in
392 case try File.read_lines path of
393 SOME (comp_thys :: incomp_thys :: fact_lines) =>
395 fun add_thy comp thy = Symtab.update (thy, comp)
396 fun add_edge_to name parent =
397 Graph.default_node (parent, ())
398 #> Graph.add_edge (parent, name)
399 fun add_fact_line line =
400 case extract_query line of
401 ("", _) => I (* shouldn't happen *)
403 Graph.default_node (name, ())
404 #> fold (add_edge_to name) parents
406 Symtab.empty |> fold (add_thy true) (unescape_metas comp_thys)
407 |> fold (add_thy false) (unescape_metas incomp_thys)
409 try_graph ctxt "loading state" Graph.empty (fn () =>
410 Graph.empty |> fold add_fact_line fact_lines)
411 in {thys = thys, fact_graph = fact_graph} end
415 fun mash_save ({thys, fact_graph, ...} : mash_state) =
417 val path = mash_state_path ()
418 val thys = Symtab.dest thys
419 val line_for_thys = escape_metas o AList.find (op =) thys
420 fun fact_line_for name parents =
421 escape_meta name ^ ": " ^ escape_metas parents
422 val append_fact = File.append path o suffix "\n" oo fact_line_for
424 File.write path (line_for_thys true ^ "\n" ^ line_for_thys false ^ "\n");
425 Graph.fold (fn (name, ((), (parents, _))) => fn () =>
426 append_fact name (Graph.Keys.dest parents))
431 Synchronized.var "Sledgehammer_Filter_MaSh.global_state" (false, empty_state)
435 fun mash_map ctxt f =
436 Synchronized.change global_state (mash_load ctxt ##> (f #> tap mash_save))
439 Synchronized.change_result global_state (mash_load ctxt #> `snd)
441 fun mash_reset ctxt =
442 Synchronized.change global_state (fn _ =>
443 (mash_RESET ctxt; File.write (mash_state_path ()) "";
444 (true, empty_state)))
448 fun mash_could_suggest_facts () = mash_home () <> ""
449 fun mash_can_suggest_facts ctxt =
450 not (Graph.is_empty (#fact_graph (mash_get ctxt)))
452 fun parents_wrt_facts ctxt facts fact_graph =
454 val graph_facts = Symtab.make (map (rpair ()) (Graph.keys fact_graph))
456 try_graph ctxt "when computing ancestor facts" [] (fn () =>
457 [] |> fold (cons o Thm.get_name_hint o snd) facts
458 |> filter (Symtab.defined graph_facts)
459 |> Graph.all_preds fact_graph)
460 val facts = Symtab.empty |> fold (fn name => Symtab.update (name, ())) facts
462 try_graph ctxt "when computing parent facts" [] (fn () =>
463 fact_graph |> Graph.restrict (Symtab.defined facts) |> Graph.maximals)
466 (* Generate more suggestions than requested, because some might be thrown out
467 later for various reasons and "meshing" gives better results with some
469 fun max_suggs_of max_facts = max_facts + Int.min (200, max_facts)
471 fun is_fact_in_graph fact_graph (_, th) =
472 can (Graph.get_node fact_graph) (Thm.get_name_hint th)
474 fun mash_suggest_facts ctxt ({overlord, ...} : params) prover max_facts hyp_ts
477 val thy = Proof_Context.theory_of ctxt
478 val fact_graph = #fact_graph (mash_get ctxt)
479 val parents = parents_wrt_facts ctxt facts fact_graph
480 val feats = features_of ctxt prover thy General (concl_t :: hyp_ts)
482 if Graph.is_empty fact_graph then []
483 else mash_QUERY ctxt overlord (max_suggs_of max_facts) (parents, feats)
484 val selected = facts |> suggested_facts suggs
485 val unknown = facts |> filter_out (is_fact_in_graph fact_graph)
486 in (selected, unknown) end
488 fun add_thys_for thy =
489 let fun add comp thy = Symtab.update (Context.theory_name thy, comp) in
490 add false thy #> fold (add true) (Theory.ancestors_of thy)
493 fun update_fact_graph ctxt (name, parents, feats, deps) (upds, graph) =
495 fun maybe_add_from from (accum as (parents, graph)) =
496 try_graph ctxt "updating graph" accum (fn () =>
497 (from :: parents, Graph.add_edge_acyclic (from, name) graph))
498 val graph = graph |> Graph.default_node (name, ())
499 val (parents, graph) = ([], graph) |> fold maybe_add_from parents
500 val (deps, graph) = ([], graph) |> fold maybe_add_from deps
501 in ((name, parents, feats, deps) :: upds, graph) end
503 val pass1_learn_timeout_factor = 0.5
505 (* The timeout is understood in a very slack fashion. *)
506 fun mash_learn_thy ctxt ({provers, verbose, overlord, ...} : params) thy timeout
509 val timer = Timer.startRealTimer ()
510 val prover = hd provers
512 Time.> (Timer.checkRealTimer timer, time_mult frac timeout)
513 val {fact_graph, ...} = mash_get ctxt
515 facts |> filter_out (is_fact_in_graph fact_graph)
516 |> sort (thm_ord o pairself snd)
518 if null new_facts then
522 val ths = facts |> map snd
524 ths |> filter_out is_likely_tautology_or_too_meta
525 |> map (rpair () o Thm.get_name_hint)
527 fun do_fact _ (accum as (_, true)) = accum
528 | do_fact ((_, (_, status)), th) ((parents, upds), false) =
530 val name = Thm.get_name_hint th
531 val feats = features_of ctxt prover thy status [prop_of th]
532 val deps = isabelle_dependencies_of all_names th
533 val upd = (name, parents, feats, deps)
534 in (([name], upd :: upds), timed_out pass1_learn_timeout_factor) end
535 val parents = parents_wrt_facts ctxt facts fact_graph
537 ((parents, []), false) |> fold do_fact new_facts |>> apsnd rev
539 fun trans {thys, fact_graph} =
541 val mash_INIT_or_ADD =
542 if Graph.is_empty fact_graph then mash_INIT else mash_ADD
543 val (upds, fact_graph) =
544 ([], fact_graph) |> fold (update_fact_graph ctxt) upds
546 (mash_INIT_or_ADD ctxt overlord (rev upds);
547 {thys = thys |> add_thys_for thy,
548 fact_graph = fact_graph})
553 "Processed " ^ string_of_int n ^ " proof" ^ plural_s n ^
555 " in " ^ string_from_time (Timer.checkRealTimer timer)
563 fun mash_learn_proof ctxt ({provers, overlord, ...} : params) t facts used_ths =
565 val thy = Proof_Context.theory_of ctxt
566 val prover = hd provers
567 val name = ATP_Util.timestamp () ^ " " ^ serial_string () (* fresh enough *)
568 val feats = features_of ctxt prover thy General [t]
569 val deps = used_ths |> map Thm.get_name_hint
571 mash_map ctxt (fn {thys, fact_graph} =>
573 val parents = parents_wrt_facts ctxt facts fact_graph
574 val upds = [(name, parents, feats, deps)]
575 val (upds, fact_graph) =
576 ([], fact_graph) |> fold (update_fact_graph ctxt) upds
578 mash_ADD ctxt overlord upds;
579 {thys = thys, fact_graph = fact_graph}
583 (* The threshold should be large enough so that MaSh doesn't kick in for Auto
584 Sledgehammer and Try. *)
585 val min_secs_for_learning = 15
586 val learn_timeout_factor = 2.0
588 fun relevant_facts ctxt (params as {learn, fact_filter, timeout, ...}) prover
589 max_facts ({add, only, ...} : fact_override) hyp_ts concl_t facts =
590 if not (subset (op =) (the_list fact_filter, fact_filters)) then
591 error ("Unknown fact filter: " ^ quote (the fact_filter) ^ ".")
594 else if max_facts <= 0 orelse null facts then
598 val thy = Proof_Context.theory_of ctxt
600 if not learn orelse Async_Manager.has_running_threads MaShN then
602 else if Time.toSeconds timeout >= min_secs_for_learning then
604 val soft_timeout = time_mult learn_timeout_factor timeout
605 val hard_timeout = time_mult 4.0 soft_timeout
606 val birth_time = Time.now ()
607 val death_time = Time.+ (birth_time, hard_timeout)
608 val desc = ("machine learner for Sledgehammer", "")
610 Async_Manager.launch MaShN birth_time death_time desc
612 (true, mash_learn_thy ctxt params thy soft_timeout facts))
618 SOME ff => (() |> ff <> iterN ? maybe_learn; ff)
620 if mash_can_suggest_facts ctxt then (maybe_learn (); meshN)
621 else if mash_could_suggest_facts () then (maybe_learn (); iterN)
623 val add_ths = Attrib.eval_thms ctxt add
624 fun prepend_facts ths accepts =
625 ((facts |> filter (member Thm.eq_thm_prop ths o snd)) @
626 (accepts |> filter_out (member Thm.eq_thm_prop ths o snd)))
629 iterative_relevant_facts ctxt params prover max_facts NONE hyp_ts
632 mash_suggest_facts ctxt params prover max_facts hyp_ts concl_t facts
634 [] |> (if fact_filter <> mashN then cons (iter (), []) else I)
635 |> (if fact_filter <> iterN then cons (mash ()) else I)
637 mesh_facts max_facts mess
638 |> not (null add_ths) ? prepend_facts add_ths
641 fun kill_learners () = Async_Manager.kill_threads MaShN "learner"
642 fun running_learners () = Async_Manager.running_threads MaShN "learner"