import numpy as np
import pandas as pd
import matplotlib
import os
from scipy import interpolate
from PyQt5.QtCore import QObject, pyqtSignal
import cfatools.processor.flow as flow
import cfatools.processor.collector as collector
import cfatools.processor.encoder as encoder
from cfatools.logreader.dataset import DatasetReader
matplotlib.use('TkAgg')
class AnalyseController(QObject):
sig_simple_step_finished = pyqtSignal(str, name="simple_step_finished")
sig_action_step_finished = pyqtSignal(str, str, str, name="action_step_finished")
def __init__(self):
super(AnalyseController, self).__init__()
self.current_dataset = None
self.volume_dict = dict()
self.encoder_df = pd.DataFrame()
self.iceblock_df = pd.DataFrame()
self.conduct_df = pd.DataFrame()
self.pump_df = pd.DataFrame()
self.picarro_df = pd.DataFrame()
self.collector_df = pd.DataFrame()
self.arrival_df_raw = pd.DataFrame()
self.arrival_df = pd.DataFrame()
def get_dataset_reader(self, dataset_path: str) -> DatasetReader:
root_directory = os.path.dirname(dataset_path)
dataset_name = os.path.basename(dataset_path)
valid, error_msg = DatasetReader.dataset_name_is_valid(dataset_name)
if not valid:
raise ValueError(error_msg)
self.current_dataset = DatasetReader(
base_path=root_directory, dataset=dataset_name
)
self.sig_simple_step_finished.emit("Dataset [" + self.current_dataset.dataset_name + "] loaded.")
return self.current_dataset
def analyse(self, override_arrival_pkl: bool = False):
dataset = self.current_dataset
self.encoder_df, self.iceblock_df, self.conduct_df, self.pump_df, self.picarro_df, self.collector_df = \
flow.get_datasets_data(dataset)
self.__apply_special_corrections__()
# Dataset's "processed" directory
processed_dir = os.path.join(dataset.dataset_path, "processed")
if not os.path.exists(processed_dir):
os.mkdir(processed_dir)
self.sig_simple_step_finished.emit("New 'processed' subdirectory created in " + dataset.dataset_path)
self.collector_df = collector.renumber_flasks(self.collector_df)
self.sig_action_step_finished.emit("Collector's flasks renumbered.", "View", "view_collector_df")
################################################################################################################
# Tubing volumes
self.volumes_dict = flow.get_tubing_volume_dict("../config/tubing_volumes.csv",
max_datetime=self.encoder_df.index[0])
self.sig_action_step_finished.emit("Volume table loaded.", "View", "view_volume_dict")
################################################################################################################
# Arrival datetimes
if override_arrival_pkl:
pkl_filepath = os.path.join(dataset.dataset_path, "binary", "arrival_nearest.pkl")
if os.path.exists(pkl_filepath):
os.remove(pkl_filepath)
self.sig_simple_step_finished.emit("Previous arrival_nearest.pkl file deleted.")
self.arrival_df_raw = flow.get_arrival_df(self.encoder_df, self.pump_df, self.volumes_dict,
parallel=False, dataset=dataset, pkl_name="arrival_nearest.pkl")
if override_arrival_pkl:
self.sig_action_step_finished.emit("Arrival_df computed.", "View", "view_arrival_df")
else:
self.sig_action_step_finished.emit("Arrival_df loaded from binary file.", "View", "view_arrival_df_raw")
self.arrival_df = self.arrival_df_raw.dropna()
self.arrival_df = flow.add_iceblock_info(self.arrival_df, self.iceblock_df)
self.arrival_df = flow.add_melted_height(self.arrival_df, self.encoder_df, self.iceblock_df)
self.arrival_df = flow.add_flask_info(self.arrival_df, self.collector_df)
self.sig_action_step_finished.emit("Arrival_df augmented with iceblock, encoder and flask info.",
"View", "view_arrival_df")
return
# Save as CSV
arrival_filepath = os.path.join(processed_dir, dataset.dataset_name + "_arrival_df.csv")
self.arrival_df.to_csv(arrival_filepath, date_format="%Y-%m-%d %H:%M:%S")
self.sig_simple_step_finished.emit("Arrival_df saved as CSV in " + arrival_filepath)
################################################################################################################
# Conducti rescaled by melting time
conduct_rescaled_df = flow.get_conduct_by_melt_time(self.arrival_df, self.conduct_df, export_in_dataset=dataset)
################################################################################################################
# Picarro rescaled by melting time
if not self.picarro_df.index.is_monotonic_increasing:
self.picarro_df.reset_index(inplace=True)
self.picarro_df['diff_time'] = self.picarro_df["datetime"] - self.picarro_df["datetime"].shift(1)
self.picarro_df = self.picarro_df.loc[
self.picarro_df.index > self.picarro_df.loc[self.picarro_df["diff_time"]
.dt.total_seconds() < 0].index[0]]
self.picarro_df = self.picarro_df.set_index("datetime")
self.picarro_df = self.picarro_df.drop(columns={'diff_time'})
picarro_rescaled_df = flow.get_picarro_by_melt_time(self.arrival_df, self.picarro_df, export_in_dataset=dataset)
################################################################################################################
# Flask
flask_df = flow.get_flask_content(self.arrival_df)
# Save as CSV
flask_df_filepath = os.path.join(processed_dir, dataset.dataset_name + "_flask_df.csv")
flask_df.to_csv(flask_df_filepath, index=False)
# Plot mm per flask
if not all(flask_df["flask"] == 0):
flask_df["mm_diff"] = flask_df["max"] - flask_df["min"]
mm_per_flask_df = flask_df[["flask", "mm_diff"]].groupby("flask").agg(sum)
mm_per_flask_df.plot.bar()
mm_per_flask_df.plot.hist(bins=round(len(mm_per_flask_df.index)/3))
################################################################################################################
# Conducti
# self._conduct_df.plot()
################################################################################################################
# Picarro & conducti vs. melted height
pic_conduc_height_filepath = os.path.join(processed_dir, dataset.dataset_name + "_pic_conduc_height.csv")
pic_conduc_height_df = self.arrival_df[["icbk_code", "icbk_name", "melted_height", "melted_height_icbk"]]
pic_conduc_height_df = pd.merge(pic_conduc_height_df, picarro_rescaled_df, left_index=True, right_index=True)
pic_conduc_height_df = pd.merge(pic_conduc_height_df, conduct_rescaled_df, left_index=True, right_index=True)
pic_conduc_height_df.to_csv(pic_conduc_height_filepath)
################################################################################################################
# Picarro
self.arrival_df = self.arrival_df[self.arrival_df["icbk_code"] > 0]
picarro2_df = pd.merge_asof(left=self.arrival_df[["icbk_code", "icbk_name", "melted_height", "picarro"]],
right=self.picarro_df,
left_on="picarro",
right_index=True)
picarro2_df["iceblock"] = picarro2_df["icbk_code"].astype(str) + '-' + picarro2_df["icbk_name"]
# print(gg.ggplot()
# + gg.geom_line(data=picarro2_df,
# mapping=gg.aes(x='picarro', y='Delta_18_16', color="iceblock"))
# )
#
# print(gg.ggplot()
# + gg.geom_path(data=picarro2_df,
# mapping=gg.aes(x='Delta_18_16', y='melted_height', color="iceblock"))
# )
################################################################################################################
# Calibrate
calib_name = "cal1"
calibrated_dir = os.path.join(dataset.dataset_path, "calibrated")
if not os.path.exists(calibrated_dir):
os.mkdir(calibrated_dir)
calib_filepath = os.path.join(calibrated_dir, dataset.dataset_name + "_" + calib_name + ".csv")
if not os.path.exists(calib_filepath):
template_calib_df = self.iceblock_df[["id", "name", "initial_height"]].copy()
template_calib_df["start"] = 0
template_calib_df = template_calib_df\
.melt(["id", "name"])\
.sort_values(["id", "value"])\
.drop(columns=["id", "variable"])
template_calib_df = template_calib_df.rename(columns={"name": "icbk_name",
"initial_height": "melted_height"})
template_calib_df[["depth",
"Delta_18_16_slope", "Delta_18_16_intercept",
"Delta_D_H_slope", "Delta_D_H_intercept"]] = [np.NaN, np.NaN, np.NaN, np.NaN, np.NaN]
template_calib_df.to_csv(calib_filepath, index=False, float_format='%.2f')
raise ValueError("Calib file not found: " + calib_filepath)
calib_df = pd.read_csv(calib_filepath, sep=",")
################################################################################################################
# Calibrate pic_conduct_height
# Absolute depth calibration
for icbk_name in calib_df["icbk_name"].unique():
icbk_calib_df = calib_df.loc[calib_df["icbk_name"] == icbk_name]
depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"],
y=icbk_calib_df["depth"])
try:
pic_conduc_height_df.loc[pic_conduc_height_df["icbk_name"] == icbk_name, "depth"] \
= depth_interp_func(pic_conduc_height_df
.loc[pic_conduc_height_df["icbk_name"] == icbk_name]["melted_height_icbk"])
except:
print(icbk_name)
# Picarro calibration
if len(calib_df.groupby(["Delta_18_16_slope", "Delta_18_16_intercept",
"Delta_D_H_slope", "Delta_D_H_intercept"]).count()) > 1:
raise ValueError("More than one calibration set for isotope is not yet supported")
isotopic_calib_df = calib_df.iloc[0]
pic_conduc_height_df["Delta_18_16_calib"] = \
pic_conduc_height_df["Delta_18_16"] * isotopic_calib_df.Delta_18_16_slope\
+ isotopic_calib_df.Delta_18_16_intercept
pic_conduc_height_df["Delta_D_H_calib"] = \
pic_conduc_height_df["Delta_D_H"] * isotopic_calib_df.Delta_D_H_slope\
+ isotopic_calib_df.Delta_D_H_intercept
# Save as CSV
pic_conduct_calibrated_filepath = os.path.join(
calibrated_dir,
dataset.dataset_name + "_pic_conduct_calibrated_" + calib_name + ".csv")
pic_conduc_height_df.to_csv(pic_conduct_calibrated_filepath, float_format='%.3f')
################################################################################################################
# Calibrate flasks
for icbk_name in calib_df["icbk_name"].unique():
icbk_calib_df = calib_df.loc[calib_df["icbk_name"] == icbk_name]
depth_interp_func = interpolate.interp1d(x=icbk_calib_df["melted_height"],
y=icbk_calib_df["depth"])
flask_df.loc[flask_df["icbk_name"] == icbk_name, "min_depth"] \
= depth_interp_func(flask_df.loc[flask_df["icbk_name"] == icbk_name]["min"])
flask_df.loc[flask_df["icbk_name"] == icbk_name, "max_depth"] \
= depth_interp_func(flask_df.loc[flask_df["icbk_name"] == icbk_name]["max"])
flask_df["diff_depth"] = flask_df["max_depth"] - flask_df["min_depth"]
flask_calibrated_filepath = os.path.join(calibrated_dir,
dataset.dataset_name + "_flask_calibrated_" + calib_name + ".csv")
flask_df.to_csv(flask_calibrated_filepath, float_format='%.2f', index=False)
def __apply_special_corrections__(self) -> None:
"""Some datasets require some manual correction before being processed, due to errors during the recording"""
if self.current_dataset.dataset_name == "20210428_ASUMA2016_8_19sq":
t1 = pd.Timestamp("2021-04-28 11:02:50", tz="UTC")
t2 = pd.Timestamp("2021-04-28 12:57:56", tz="UTC")
t3 = pd.Timestamp("2021-04-28 13:20:28", tz="UTC")
t4 = pd.Timestamp("2021-04-28 15:19:37", tz="UTC")
self.encoder_df = pd.concat([self.encoder_df.loc[t1:t2], self.encoder_df.loc[t3:t4]])
self.sig_simple_step_finished.emit("Special correction applied on encoder_df.")
if self.current_dataset.dataset_name in ["20210826_ASUMA2016_6-1_sq", "20210827_ASUMA2016_6_11sq"]:
self.picarro_df.index = self.picarro_df.index - pd.Timedelta(115, unit="s")
self.sig_simple_step_finished.emit("Special correction applied on picarro_df.")
if self.current_dataset.dataset_name == "20220317_ASUMA2016_25_19sq":
self.encoder_df = encoder.shift_stacking_event(
self.encoder_df,
old_peak_start=pd.Timestamp("2022-03-17 13:28:27.446578+00:00"),
old_peak_end=pd.Timestamp("2022-03-17 13:28:41.721330+00:00"),
shift=pd.Timedelta(seconds=60))
self.sig_simple_step_finished.emit("Special correction applied on encoder_df.")
if self.current_dataset.dataset_name == "20220329_ASUMA2016_23_12sq":
self.encoder_df = encoder.shift_stacking_event(
self.encoder_df,
old_peak_start=pd.Timestamp("2022-03-29 16:58:11.552587+00:00"),
old_peak_end=pd.Timestamp("2022-03-29 16:58:48.006613+00:00"),
shift=pd.Timedelta(seconds=80))
self.sig_simple_step_finished.emit("Special correction applied on encoder_df.")