time_logger: resume on failure operational

lib/time_logger.ml

@@ -4,10 +4,21 @@ open Bistro_engine
 type stats = {
   elapsed_time : float ;
 }
+[@@deriving sexp]
+
+let backup_fn = "time_logger.bck"
 
 let create () = object (self)
   val started = String.Table.create ()
-  val table = String.Table.create ()
+  val mutable table = String.Table.create ()
+  method backup =
+    String.Table.sexp_of_t sexp_of_stats table
+    |> Sexp.to_string
+    |> Out_channel.write_all backup_fn
+  method load =
+    In_channel.read_all backup_fn
+    |> Sexp.of_string
+    |> String.Table.t_of_sexp stats_of_sexp
   method event config t = function
     | Scheduler.Task_started (task, _) ->
       String.Table.set started (Task.id task) t
@@ -16,20 +27,21 @@ let create () = object (self)
       let start_time = String.Table.find_exn started id in
       String.Table.set table id {
         elapsed_time = t -. start_time ;
-      }
+      } ;
+      self#backup
     | Task_ended (Input_check _ | Select_check _ | Map_command_result _)
     | Init _
     | Task_ready _
     | Task_skipped _ -> ()
 
-  method elapsed_time : 'a. 'a Bistro.workflow -> float = fun w ->
+  method elapsed_time : 'a. stats String.Table.t -> 'a Bistro.workflow -> float = fun table w ->
     (String.Table.find_exn table (Bistro.Workflow.id w)).elapsed_time
-  method elapsed_time_of_result : Convergence_detection.result -> string * float option =
+  method elapsed_time_of_result : stats String.Table.t -> Convergence_detection.result -> string * float option =
     let f w =
       let id = Bistro.Workflow.id w in
       id, Option.map (String.Table.find table (Bistro.Workflow.id w)) ~f:(fun x -> x.elapsed_time)
     in
-    function
+    fun table -> function
     | `Pcoc w
     | `Pcoc_gamma w
     | `Pcoc_C60 w -> f w
@@ -45,12 +57,13 @@ let create () = object (self)
   method wait4shutdown = Lwt.return ()
   method logger = (self :> Scheduler.logger)
   method report detection_results fn =
+    let table = self#load in
     List.map detection_results ~f:(fun (dataset_res : Convergence_detection.dataset_res) ->
         let tree_name = dataset_res.tree_prefix in
         let model_name = dataset_res.model_prefix in
         List.map dataset_res.res_by_tools ~f:(fun r ->
             let meth_name = Convergence_detection.meth_string_of_result r in
-            let id, elapsed_time = self#elapsed_time_of_result r in
+            let id, elapsed_time = self#elapsed_time_of_result table r in
             String.concat
               ~sep:"\t"
               [ tree_name ; model_name ; meth_name ; id ;
@@ -59,5 +72,8 @@ let create () = object (self)
       )
     |> List.concat
     |> Out_channel.write_lines fn
-
+  initializer
+    if Sys.file_exists backup_fn = `Yes then
+      table <- self#load
+    else self#backup
 end