Analyse. Use in-app calibration editor.

.idea/watcherTasks.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectTasksOptions">
     <TaskOptions isEnabled="true">
-      <option name="arguments" value="$FilePath$" />
+      <option name="arguments" value="$FilePath$ -l 120" />
       <option name="checkSyntaxErrors" value="true" />
       <option name="description" />
       <option name="exitCodeBehavior" value="ERROR" />

src/controller/analysecontroller.py

@@ -10,7 +10,8 @@ import cfatools.processor.flow as flow
 import cfatools.processor.collector as collector
 import cfatools.processor.encoder as encoder
 from cfatools.logreader.dataset import DatasetReader
-matplotlib.use('TkAgg')
+
+matplotlib.use("TkAgg")
 
 
 class AnalyseController(QObject):
@@ -35,6 +36,7 @@ class AnalyseController(QObject):
         self.flask_df = pd.DataFrame()
         self.flask_df_filepath = ""
         self.pic_conduc_height_filepath = ""
+        self.calib_filepath = ""
         self.calib_df = pd.DataFrame()
         self.pic_conduct_calibrated_filepath = ""
         self.flask_calibrated_filepath = ""
@@ -45,16 +47,20 @@ class AnalyseController(QObject):
         valid, error_msg = DatasetReader.dataset_name_is_valid(dataset_name)
         if not valid:
             raise ValueError(error_msg)
-        self.current_dataset = DatasetReader(
-            base_path=root_directory, dataset=dataset_name
-        )
+        self.current_dataset = DatasetReader(base_path=root_directory, dataset=dataset_name)
         self.sig_simple_step_finished.emit("Dataset [" + self.current_dataset.dataset_name + "] loaded.")
         return self.current_dataset
 
     def analyse(self, override_arrival_pkl: bool = False):
         dataset = self.current_dataset
-        self.encoder_df, self.iceblock_df, self.conduct_df, self.pump_df, self.picarro_df, self.collector_df = \
-            flow.get_datasets_data(dataset)
+        (
+            self.encoder_df,
+            self.iceblock_df,
+            self.conduct_df,
+            self.pump_df,
+            self.picarro_df,
+            self.collector_df,
+        ) = flow.get_datasets_data(dataset)
 
         self.__apply_special_corrections__()
 
@@ -69,8 +75,9 @@ class AnalyseController(QObject):
 
         ################################################################################################################
         # Tubing volumes
-        self.volumes_dict = flow.get_tubing_volume_dict("../config/tubing_volumes.csv",
-                                                   max_datetime=self.encoder_df.index[0])
+        self.volumes_dict = flow.get_tubing_volume_dict(
+            "../config/tubing_volumes.csv", max_datetime=self.encoder_df.index[0]
+        )
         self.sig_action_step_finished.emit("Volume table loaded.", "View", "view_volume_dict")
 
         ################################################################################################################
@@ -80,8 +87,14 @@ class AnalyseController(QObject):
             if os.path.exists(pkl_filepath):
                 os.remove(pkl_filepath)
                 self.sig_simple_step_finished.emit("Previous arrival_nearest.pkl file deleted.")
-        self.arrival_df_raw = flow.get_arrival_df(self.encoder_df, self.pump_df, self.volumes_dict,
-                                                  parallel=False, dataset=dataset, pkl_name="arrival_nearest.pkl")
+        self.arrival_df_raw = flow.get_arrival_df(
+            self.encoder_df,
+            self.pump_df,
+            self.volumes_dict,
+            parallel=False,
+            dataset=dataset,
+            pkl_name="arrival_nearest.pkl",
+        )
         if override_arrival_pkl:
             self.sig_action_step_finished.emit("Arrival_df computed.", "View", "view_arrival_df")
         else:
@@ -91,14 +104,20 @@ class AnalyseController(QObject):
         self.arrival_df = flow.add_iceblock_info(self.arrival_df, self.iceblock_df)
         self.arrival_df = flow.add_melted_height(self.arrival_df, self.encoder_df, self.iceblock_df)
         self.arrival_df = flow.add_flask_info(self.arrival_df, self.collector_df)
-        self.sig_action_step_finished.emit("Arrival_df augmented with iceblock, encoder and flask info.",
-                                           "View", "view_arrival_df")
+        self.sig_action_step_finished.emit(
+            "Arrival_df augmented with iceblock, encoder and flask info.",
+            "View",
+            "view_arrival_df",
+        )
 
         # Save as CSV
         self.arrival_filepath = os.path.join(processed_dir, dataset.dataset_name + "_arrival_df.csv")
         self.arrival_df.to_csv(self.arrival_filepath, date_format="%Y-%m-%d %H:%M:%S")
-        self.sig_action_step_finished.emit("Arrival_df saved as CSV in [" + self.arrival_filepath + "]",
-                                           "Open", "open_arrival_csv")
+        self.sig_action_step_finished.emit(
+            "Arrival_df saved as CSV in [" + self.arrival_filepath + "]",
+            "Open",
+            "open_arrival_csv",
+        )
 
         ################################################################################################################
         # Conducti rescaled by melting time
@@ -109,12 +128,13 @@ class AnalyseController(QObject):
         # Picarro rescaled by melting time
         if not self.picarro_df.index.is_monotonic_increasing:
             self.picarro_df.reset_index(inplace=True)
-            self.picarro_df['diff_time'] = self.picarro_df["datetime"] - self.picarro_df["datetime"].shift(1)
+            self.picarro_df["diff_time"] = self.picarro_df["datetime"] - self.picarro_df["datetime"].shift(1)
             self.picarro_df = self.picarro_df.loc[
-                self.picarro_df.index > self.picarro_df.loc[self.picarro_df["diff_time"]
-                                                            .dt.total_seconds() < 0].index[0]]
+                self.picarro_df.index
+                > self.picarro_df.loc[self.picarro_df["diff_time"].dt.total_seconds() < 0].index[0]
+            ]
             self.picarro_df = self.picarro_df.set_index("datetime")
-            self.picarro_df = self.picarro_df.drop(columns={'diff_time'})
+            self.picarro_df = self.picarro_df.drop(columns={"diff_time"})
         picarro_rescaled_df = flow.get_picarro_by_melt_time(self.arrival_df, self.picarro_df, export_in_dataset=dataset)
         self.sig_simple_step_finished.emit("Picarro-R(escaled) created.")
 
@@ -126,8 +146,11 @@ class AnalyseController(QObject):
         # Save as CSV
         self.flask_df_filepath = os.path.join(processed_dir, dataset.dataset_name + "_flask_df.csv")
         self.flask_df.to_csv(self.flask_df_filepath, index=False)
-        self.sig_action_step_finished.emit("flask_df saved as CSV in [" + self.arrival_filepath + "]",
-                                           "Open", "open_flask_csv")
+        self.sig_action_step_finished.emit(
+            "flask_df saved as CSV in [" + self.arrival_filepath + "]",
+            "Open",
+            "open_flask_csv",
+        )
 
         # Plot mm per flask
         # if not all(self.flask_df["flask"] == 0):
@@ -149,7 +172,9 @@ class AnalyseController(QObject):
         pic_conduc_height_df.to_csv(self.pic_conduc_height_filepath)
         self.sig_action_step_finished.emit(
             "pic_conduc_height_df saved as CSV in [" + self.pic_conduc_height_filepath + "]",
-            "Open", "open_pic_conduc_height_df")
+            "Open",
+            "open_pic_conduc_height_df",
+        )
 
         ################################################################################################################
         # Picarro
@@ -176,25 +201,34 @@ class AnalyseController(QObject):
         calibrated_dir = os.path.join(dataset.dataset_path, "calibrated")
         if not os.path.exists(calibrated_dir):
             os.mkdir(calibrated_dir)
-        calib_filepath = os.path.join(calibrated_dir, dataset.dataset_name + "_" + calib_name + ".csv")
-        if not os.path.exists(calib_filepath):
+        self.calib_filepath = os.path.join(calibrated_dir, dataset.dataset_name + "_" + calib_name + ".csv")
+        if not os.path.exists(self.calib_filepath):
             template_calib_df = self.iceblock_df[["id", "name", "initial_height"]].copy()
             template_calib_df["start"] = 0
-            template_calib_df = template_calib_df\
-                .melt(["id", "name"])\
-                .sort_values(["id", "value"])\
-                .drop(columns=["id", "variable"])
-            template_calib_df = template_calib_df.rename(columns={"name": "icbk_name",
-                                                                  "initial_height": "melted_height"})
-            template_calib_df[["depth",
-                               "Delta_18_16_slope", "Delta_18_16_intercept",
-                               "Delta_D_H_slope", "Delta_D_H_intercept"]] = [np.NaN, np.NaN, np.NaN, np.NaN, np.NaN]
-            template_calib_df.to_csv(calib_filepath, index=False, float_format='%.2f')
-        self.calib_df = pd.read_csv(calib_filepath, sep=",")
-
-        if len(self.calib_df.index) < 1:
-            self.sig_simple_step_finished.emit("Calibration file is empty: [" + calib_filepath + "]")
+            template_calib_df = (
+                template_calib_df.melt(["id", "name"]).sort_values(["id", "value"]).drop(columns=["id", "variable"])
+            )
+            template_calib_df = template_calib_df.rename(columns={"name": "icbk_name", "value": "melted_height"})
+            template_calib_df[
+                [
+                    "depth",
+                    "Delta_18_16_slope",
+                    "Delta_18_16_intercept",
+                    "Delta_D_H_slope",
+                    "Delta_D_H_intercept",
+                ]
+            ] = [np.NaN, np.NaN, np.NaN, np.NaN, np.NaN]
+            template_calib_df.to_csv(self.calib_filepath, index=False, float_format="%.2f", sep=";")
+        self.calib_df = pd.read_csv(self.calib_filepath, sep=";")
+
+        if all(self.calib_df["depth"].isna()):
+            self.sig_action_step_finished.emit(
+                "Calibration file is empty: [" + self.calib_filepath + "]",
+                "Edit",
+                "edit_calib_df",
+            )
             self.sig_simple_step_finished.emit("Abort calibration.")
+            return
         else:
             self.sig_action_step_finished.emit("Calibration file loaded.", "View", "view_calib_df")
 
@@ -203,57 +237,68 @@ class AnalyseController(QObject):
         # Absolute depth calibration
         for icbk_name in self.calib_df["icbk_name"].unique():
             icbk_calib_df = self.calib_df.loc[self.calib_df["icbk_name"] == icbk_name]
-            depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"],
-                                                     y=icbk_calib_df["depth"])
+            depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"], y=icbk_calib_df["depth"])
             try:
-                pic_conduc_height_df.loc[pic_conduc_height_df["icbk_name"] == icbk_name, "depth"] \
-                    = depth_interp_func(pic_conduc_height_df
-                                        .loc[pic_conduc_height_df["icbk_name"] == icbk_name]["melted_height_icbk"])
+                pic_conduc_height_df.loc[pic_conduc_height_df["icbk_name"] == icbk_name, "depth"] = depth_interp_func(
+                    pic_conduc_height_df.loc[pic_conduc_height_df["icbk_name"] == icbk_name]["melted_height_icbk"]
+                )
             except:
                 print(icbk_name)
 
         # Picarro calibration
-        if len(self.calib_df.groupby(["Delta_18_16_slope", "Delta_18_16_intercept",
-                                      "Delta_D_H_slope", "Delta_D_H_intercept"]).count()) > 1:
+        if (
+            len(
+                self.calib_df.groupby(
+                    ["Delta_18_16_slope", "Delta_18_16_intercept", "Delta_D_H_slope", "Delta_D_H_intercept"]
+                ).count()
+            )
+            > 1
+        ):
             raise ValueError("More than one calibration set for isotope is not yet supported")
 
         isotopic_calib_df = self.calib_df.iloc[0]
         calib_pic_conduc_height_df = pic_conduc_height_df.copy()
-        calib_pic_conduc_height_df["Delta_18_16_calib"] = \
-            calib_pic_conduc_height_df["Delta_18_16"] * isotopic_calib_df.Delta_18_16_slope\
+        calib_pic_conduc_height_df["Delta_18_16_calib"] = (
+            calib_pic_conduc_height_df["Delta_18_16"] * isotopic_calib_df.Delta_18_16_slope
             + isotopic_calib_df.Delta_18_16_intercept
-        calib_pic_conduc_height_df["Delta_D_H_calib"] = \
-            calib_pic_conduc_height_df["Delta_D_H"] * isotopic_calib_df.Delta_D_H_slope\
+        )
+        calib_pic_conduc_height_df["Delta_D_H_calib"] = (
+            calib_pic_conduc_height_df["Delta_D_H"] * isotopic_calib_df.Delta_D_H_slope
             + isotopic_calib_df.Delta_D_H_intercept
+        )
 
         # Save as CSV
         self.pic_conduct_calibrated_filepath = os.path.join(
-            calibrated_dir,
-            dataset.dataset_name + "_pic_conduct_calibrated_" + calib_name + ".csv")
-        calib_pic_conduc_height_df.to_csv(self.pic_conduct_calibrated_filepath, float_format='%.3f')
+            calibrated_dir, dataset.dataset_name + "_pic_conduct_calibrated_" + calib_name + ".csv"
+        )
+        calib_pic_conduc_height_df.to_csv(self.pic_conduct_calibrated_filepath, float_format="%.3f")
         self.sig_action_step_finished.emit(
             "Calibrated Picarro & Conducti saved as CSV in [" + self.pic_conduct_calibrated_filepath + "]",
-            "Open", "open_calib_pic_conduc_height_df")
+            "Open",
+            "open_calib_pic_conduc_height_df",
+        )
 
         ################################################################################################################
         # Calibrate flasks
         calib_flask_df = self.flask_df.copy()
         for icbk_name in self.calib_df["icbk_name"].unique():
             icbk_calib_df = self.calib_df.loc[self.calib_df["icbk_name"] == icbk_name]
-            depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"],
-                                                     y=icbk_calib_df["depth"])
-            calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name, "min_depth"] \
-                = depth_interp_func(calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name]["min"])
-            calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name, "max_depth"] \
-                = depth_interp_func(calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name]["max"])
+            depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"], y=icbk_calib_df["depth"])
+            calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name, "min_depth"] = depth_interp_func(
+                calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name]["min"]
+            )
+            calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name, "max_depth"] = depth_interp_func(
+                calib_flask_df.loc[calib_flask_df["icbk_name"] == icbk_name]["max"]
+            )
         calib_flask_df["diff_depth"] = calib_flask_df["max_depth"] - calib_flask_df["min_depth"]
 
-        self.flask_calibrated_filepath = os.path.join(calibrated_dir,
-                                                 dataset.dataset_name + "_flask_calibrated_" + calib_name + ".csv")
-        calib_flask_df.to_csv(self.flask_calibrated_filepath, float_format='%.2f', index=False)
+        self.flask_calibrated_filepath = os.path.join(
+            calibrated_dir, dataset.dataset_name + "_flask_calibrated_" + calib_name + ".csv"
+        )
+        calib_flask_df.to_csv(self.flask_calibrated_filepath, float_format="%.2f", index=False)
         self.sig_action_step_finished.emit(
-            "Calibrated Flask saved as CSV in [" + self.flask_calibrated_filepath + "]",
-            "Open", "open_calib_flask_df")
+            "Calibrated Flask saved as CSV in [" + self.flask_calibrated_filepath + "]", "Open", "open_calib_flask_df"
+        )
 
     def __apply_special_corrections__(self) -> None:
         """Some datasets require some manual correction before being processed, due to errors during the recording"""
@@ -274,7 +319,8 @@ class AnalyseController(QObject):
                 self.encoder_df,
                 old_peak_start=pd.Timestamp("2022-03-17 13:28:27.446578+00:00"),
                 old_peak_end=pd.Timestamp("2022-03-17 13:28:41.721330+00:00"),
-                shift=pd.Timedelta(seconds=60))
+                shift=pd.Timedelta(seconds=60),
+            )
             self.sig_simple_step_finished.emit("Special correction applied on encoder_df.")
 
         if self.current_dataset.dataset_name == "20220329_ASUMA2016_23_12sq":
@@ -282,9 +328,13 @@ class AnalyseController(QObject):
                 self.encoder_df,
                 old_peak_start=pd.Timestamp("2022-03-29 16:58:11.552587+00:00"),
                 old_peak_end=pd.Timestamp("2022-03-29 16:58:48.006613+00:00"),
-                shift=pd.Timedelta(seconds=80))
+                shift=pd.Timedelta(seconds=80),
+            )
             self.sig_simple_step_finished.emit("Special correction applied on encoder_df.")
 
+    def update_calib_csv(self, calib_df: pd.DataFrame):
+        calib_df.to_csv(self.calib_filepath, index=False, float_format="%.5f", sep=";")
+
 
 class Worker(QObject):
     """Worker class used to move long "analyse" computations to another thread, to avoid GUI freezing."""
@@ -296,5 +346,8 @@ class Worker(QObject):
         self.ctrl = controller
 
     def run(self):
-        self.ctrl.analyse()
+        try:
+            self.ctrl.analyse()
+        except Exception as e:
+            print(e)
         self.finished.emit()

src/uim/analyseuim.py

@@ -15,7 +15,11 @@ class AnalyseUim(QObject):
     """Class managing the Widget which are common for all Picarro calib sub-tabs."""
 
     def __init__(
-        self, ctrl: AnalyseController, main_ui: Ui_MainWindow, config: Config, main_window
+        self,
+        ctrl: AnalyseController,
+        main_ui: Ui_MainWindow,
+        config: Config,
+        main_window,
     ):
         super(AnalyseUim, self).__init__()
         self.main_ui = main_ui
@@ -25,7 +29,9 @@ class AnalyseUim(QObject):
 
         self.__init_tableview__()
 
-        self.main_ui.analyse_pushbutton_dataset_load.clicked.connect(self.__load_dataset__)
+        self.main_ui.analyse_pushbutton_dataset_load.clicked.connect(
+            self.__load_dataset__
+        )
         self.main_ui.analyse_toolbutton_run.clicked.connect(self.__run__)
         self.ctrl.sig_simple_step_finished.connect(self.__display_simple_step__)
         self.ctrl.sig_action_step_finished.connect(self.__display_step_with_button__)
@@ -42,7 +48,9 @@ class AnalyseUim(QObject):
             utils.show_popup(str(e), "Could not load data set!")
             return
 
-        self.main_ui.analyse_lineedit_dataset.setText(self.ctrl.current_dataset.dataset_name)
+        self.main_ui.analyse_lineedit_dataset.setText(
+            self.ctrl.current_dataset.dataset_name
+        )
 
     def __run__(self):
         """Run analyse, in another thread to prevent GUI freezing."""
@@ -54,7 +62,9 @@ class AnalyseUim(QObject):
         self.worker.finished.connect(self.thread.quit)
         self.worker.finished.connect(self.worker.deleteLater)
         self.thread.finished.connect(self.thread.deleteLater)
-        self.worker.finished.connect(lambda: self.main_ui.analyse_toolbutton_run.setEnabled(True))
+        self.worker.finished.connect(
+            lambda: self.main_ui.analyse_toolbutton_run.setEnabled(True)
+        )
         self.thread.start()
 
     def __init_tableview__(self):
@@ -66,7 +76,9 @@ class AnalyseUim(QObject):
         table, row_position = self.__add_row_in_table__()
         table.setItem(row_position, 0, QTableWidgetItem(message))
 
-    def __display_step_with_button__(self, message: str, button_text: str, button_action: str):
+    def __display_step_with_button__(
+        self, message: str, button_text: str, button_action: str
+    ):
         table, row_position = self.__add_row_in_table__()
         table.setItem(row_position, 0, QTableWidgetItem(message))
         button = QPushButton(button_text)
@@ -106,6 +118,9 @@ class AnalyseUim(QObject):
     def __open_pic_conduc_height_df__(self):
         os.system("xdg-open " + self.ctrl.pic_conduc_height_filepath)
 
+    def __edit_calib_df__(self):
+        self.__edit_dataframe_in_qdialog__(self.ctrl.calib_df)
+
     def __view_calib_df__(self):
         self.__view_dataframe_in_qdialog__(self.ctrl.calib_df)
 
@@ -125,9 +140,22 @@ class AnalyseUim(QObject):
         my_dialog.setLayout(layout)
         my_dialog.show()
 
+    def __edit_dataframe_in_qdialog__(self, dataframe: pd.DataFrame):
+        self.model = EditableTableModel(dataframe)
+        self.model.dataChanged.connect(self.__update_model_df__)
+        self.table = QTableView()
+        self.table.setModel(self.model)
+        my_dialog = QDialog(self.main_window)
+        layout = QVBoxLayout()
+        layout.addWidget(self.table)
+        my_dialog.setLayout(layout)
+        my_dialog.show()
 
-class TableModel(QAbstractTableModel):
+    def __update_model_df__(self):
+        self.ctrl.update_calib_csv(self.model.getData())
 
+
+class TableModel(QAbstractTableModel):
     def __init__(self, data):
         super(TableModel, self).__init__()
         self._data = data
@@ -150,4 +178,36 @@ class TableModel(QAbstractTableModel):
                 return str(self._data.columns[section])
 
             if orientation == Qt.Vertical:
-                return str(self._data.index[section])
+                return str(self._data.index[section])
\ No newline at end of file
+
+
+class EditableTableModel(TableModel):
+    def __init__(self, data):
+        super(EditableTableModel, self).__init__(data)
+
+    def getData(self):
+        return self._data
+
+    def setData(self, index, value, role):
+        if not index.isValid():
+            return False
+        if role != Qt.EditRole:
+            return False
+        row = index.row()
+        if row < 0 or row >= len(self._data.values):
+            return False
+        column = index.column()
+        if column < 0 or column >= self._data.columns.size:
+            return False
+        self._data.iloc[row, column] = value
+        self.dataChanged.emit(index, index)
+        return True
+
+    def flags(self, index):
+        flags = super(self.__class__, self).flags(index)
+        flags |= Qt.ItemIsEditable
+        flags |= Qt.ItemIsSelectable
+        flags |= Qt.ItemIsEnabled
+        flags |= Qt.ItemIsDragEnabled
+        flags |= Qt.ItemIsDropEnabled
+        return flags