From 638508ca80410622be1d87ab3bd9736c1c7c22ea Mon Sep 17 00:00:00 2001
From: Olivier Jossoud <olivier.jossoud@lsce.ipsl.fr>
Date: Mon, 14 Oct 2019 16:14:15 +0200
Subject: [PATCH] Explo. Color, offset, mult_factor, timeshift and X/Y original
OK.
---
pyqt/mainwindow.ui | 20 ++-
src/dataprovider/exploprovider.py | 6 +-
src/gui/uimainwindow.py | 20 ++-
src/uim/explouim.py | 258 ++++++++++++++++++++++++------
4 files changed, 239 insertions(+), 65 deletions(-)
diff --git a/pyqt/mainwindow.ui b/pyqt/mainwindow.ui
index 780ec8b..4122394 100644
--- a/pyqt/mainwindow.ui
+++ b/pyqt/mainwindow.ui
@@ -61,17 +61,17 @@
</column>
<column>
<property name="text">
- <string>Vertical line</string>
+ <string>Step curve</string>
</property>
</column>
<column>
<property name="text">
- <string>Cursor</string>
+ <string>Vertical line</string>
</property>
</column>
<column>
<property name="text">
- <string>Curve</string>
+ <string>Cursor</string>
</property>
</column>
<column>
@@ -91,12 +91,22 @@
</column>
<column>
<property name="text">
- <string>Time shift</string>
+ <string>Time shift (s)</string>
+ </property>
+ </column>
+ <column>
+ <property name="text">
+ <string>Visible</string>
+ </property>
+ </column>
+ <column>
+ <property name="text">
+ <string>X (orig.)</string>
</property>
</column>
<column>
<property name="text">
- <string>Delete</string>
+ <string>Y (orig.)</string>
</property>
</column>
</widget>
diff --git a/src/dataprovider/exploprovider.py b/src/dataprovider/exploprovider.py
index ea0dc05..0dec51d 100644
--- a/src/dataprovider/exploprovider.py
+++ b/src/dataprovider/exploprovider.py
@@ -61,7 +61,7 @@ class Dataset:
# Get dataset name
self.dataset_text = directory_name[-9:]
- self.last_data_datetime = datetime.datetime(1970, 1, 1)
+ self.last_data_datetime = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
self.instlogs = {}
@@ -118,6 +118,7 @@ class InstrumentInstantLog(InstrumentLog):
def __get_df__(self) -> pd.DataFrame:
df = pd.read_csv(self.full_file_name, sep=",", parse_dates=["datetime"])
+ df["datetime"] = df["datetime"].dt.tz_localize('UTC')
return df
def get_variables(self):
@@ -125,7 +126,6 @@ class InstrumentInstantLog(InstrumentLog):
def get_timeseries(self, variable: str) -> pd.DataFrame:
timeseries_df = self.df[self.df["name"] == variable]
- timeseries_df.rename(columns={'value': variable}, inplace=True)
timeseries_df = timeseries_df.drop(columns=['name'])
return timeseries_df
@@ -137,6 +137,7 @@ class InstrumentPeriodicLog(InstrumentLog):
def __get_df__(self) -> pd.DataFrame:
df = pd.read_csv(self.full_file_name, sep="\t", parse_dates=["datetime"])
+ df["datetime"] = df["datetime"].dt.tz_localize('UTC')
return df
def get_variables(self):
@@ -146,4 +147,5 @@ class InstrumentPeriodicLog(InstrumentLog):
def get_timeseries(self, variable: str) -> pd.DataFrame:
timeseries_df = self.df[["datetime", variable]]
+ timeseries_df.rename(columns={variable: 'value'}, inplace=True)
return timeseries_df
diff --git a/src/gui/uimainwindow.py b/src/gui/uimainwindow.py
index 486d78a..7b0c14e 100644
--- a/src/gui/uimainwindow.py
+++ b/src/gui/uimainwindow.py
@@ -25,7 +25,7 @@ class Ui_MainWindow(object):
self.explo_tablewidget_variables = QtWidgets.QTableWidget(self.tab_explo)
self.explo_tablewidget_variables.setGeometry(QtCore.QRect(20, 50, 1021, 201))
self.explo_tablewidget_variables.setObjectName("explo_tablewidget_variables")
- self.explo_tablewidget_variables.setColumnCount(10)
+ self.explo_tablewidget_variables.setColumnCount(12)
self.explo_tablewidget_variables.setRowCount(0)
item = QtWidgets.QTableWidgetItem()
self.explo_tablewidget_variables.setHorizontalHeaderItem(0, item)
@@ -47,6 +47,10 @@ class Ui_MainWindow(object):
self.explo_tablewidget_variables.setHorizontalHeaderItem(8, item)
item = QtWidgets.QTableWidgetItem()
self.explo_tablewidget_variables.setHorizontalHeaderItem(9, item)
+ item = QtWidgets.QTableWidgetItem()
+ self.explo_tablewidget_variables.setHorizontalHeaderItem(10, item)
+ item = QtWidgets.QTableWidgetItem()
+ self.explo_tablewidget_variables.setHorizontalHeaderItem(11, item)
self.horizontalLayoutWidget_2 = QtWidgets.QWidget(self.tab_explo)
self.horizontalLayoutWidget_2.setGeometry(QtCore.QRect(20, 10, 221, 31))
self.horizontalLayoutWidget_2.setObjectName("horizontalLayoutWidget_2")
@@ -183,11 +187,11 @@ class Ui_MainWindow(object):
item = self.explo_tablewidget_variables.horizontalHeaderItem(1)
item.setText(_translate("MainWindow", "Variable"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(2)
- item.setText(_translate("MainWindow", "Vertical line"))
+ item.setText(_translate("MainWindow", "Step curve"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(3)
- item.setText(_translate("MainWindow", "Cursor"))
+ item.setText(_translate("MainWindow", "Vertical line"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(4)
- item.setText(_translate("MainWindow", "Curve"))
+ item.setText(_translate("MainWindow", "Cursor"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(5)
item.setText(_translate("MainWindow", "Color"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(6)
@@ -195,9 +199,13 @@ class Ui_MainWindow(object):
item = self.explo_tablewidget_variables.horizontalHeaderItem(7)
item.setText(_translate("MainWindow", "Mult. factor"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(8)
- item.setText(_translate("MainWindow", "Time shift"))
+ item.setText(_translate("MainWindow", "Time shift (s)"))
item = self.explo_tablewidget_variables.horizontalHeaderItem(9)
- item.setText(_translate("MainWindow", "Delete"))
+ item.setText(_translate("MainWindow", "Visible"))
+ item = self.explo_tablewidget_variables.horizontalHeaderItem(10)
+ item.setText(_translate("MainWindow", "X (orig.)"))
+ item = self.explo_tablewidget_variables.horizontalHeaderItem(11)
+ item.setText(_translate("MainWindow", "Y (orig.)"))
self.explo_label_dataset.setText(_translate("MainWindow", "Dataset:"))
self.explo_pushbutton_add_row.setText(_translate("MainWindow", "+ Add Row"))
self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab_explo), _translate("MainWindow", "Data exploration"))
diff --git a/src/uim/explouim.py b/src/uim/explouim.py
index 53f838c..ad8b48a 100644
--- a/src/uim/explouim.py
+++ b/src/uim/explouim.py
@@ -1,9 +1,11 @@
import datetime
import pyqtgraph as pg
from PyQt5.QtWidgets import *
+from PyQt5.QtGui import QColor
from PyQt5.QtCore import *
import pandas as pd
import numpy as np
+from pyqtgraph.GraphicsScene.mouseEvents import MouseClickEvent
import utils
from config import Config
@@ -16,11 +18,26 @@ class ExploUim:
# "Variables" tableWidget columns identifiers.
INSTRUMENT_COL = 0
VARIABLE_COL = 1
- VLINE_COL = 2
- CURSOR_COL = 3
- CURVE_COL = 4
- COLOR_COL = 5
- DELETE_COL = 6
+ CURVE_COL = 2
+ VLINE_COL = 3
+ CURSOR_COL = 4
+ COLOR_COL = 5
+ OFFSET_COL = 6
+ MULT_COL = 7
+ TIMESHIFT_COL = 8
+ VISIBLE_COL = 9
+ XORIG_COL = 10
+ YORIG_COL = 11
+
+ DEFAULT_COLORS = [QColor(255, 255, 255), # White
+ QColor(228, 26, 28), # Red
+ QColor(55, 126, 184), # Blue
+ QColor(77, 175, 74), # Green
+ QColor(152, 78, 163), # Violet
+ QColor(255, 127, 0), # Orange
+ QColor(255, 255, 51), # Yellow
+ QColor(166, 86, 40), # Brown
+ QColor(247, 129, 191)] # Pink
def __init__(self, explo_prvd: ExploProvider, main_ui: Ui_MainWindow, config: Config):
self.main_ui = main_ui
@@ -29,10 +46,17 @@ class ExploUim:
self.current_dataset = None
+ # The "var_id" is used to identify and match table's lines and plot's items (curves, etc.). Table's row id can
+ # not be used because it changes when previous rows are deleted.
+ self.current_variable_id = 0
+
self.__initialize_variable_table__()
self.__initialize_plot__()
self.__initialize_dataset_combobox__()
+ self.main_ui.explo_pushbutton_add_row.clicked.connect(self.__add_new_row_in_variable_table__)
+ self.main_ui.explo_tablewidget_variables.cellClicked.connect(self.__change_color__)
+
def __initialize_dataset_combobox__(self):
"""Populate the "datasets" combobox with the existing dataset directory names."""
data_root_dir = self.config.read("DATA_SOURCE", "absolute_root_dir")
@@ -45,7 +69,8 @@ class ExploUim:
def __update_current_dataset__(self, dataset_dir: str):
self.current_dataset = self.explo_prvd.datasets[dataset_dir]
- self.__update_instruments_comboboxes__()
+ for row_id in range(self.main_ui.explo_tablewidget_variables.rowCount()):
+ self.__update_instruments_combobox__(row_id)
####################################################################################################################
# "Variables" table
@@ -54,58 +79,100 @@ class ExploUim:
"""Initialize the table containing the to-be-displayed variables"""
# Set column widths
- self.main_ui.explo_tablewidget_variables.setColumnWidth(self.INSTRUMENT_COL, 90)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.INSTRUMENT_COL, 110)
self.main_ui.explo_tablewidget_variables.setColumnWidth(self.VARIABLE_COL, 90)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.CURVE_COL, 85)
self.main_ui.explo_tablewidget_variables.setColumnWidth(self.VLINE_COL, 95)
- self.main_ui.explo_tablewidget_variables.setColumnWidth(self.CURSOR_COL, 80)
- self.main_ui.explo_tablewidget_variables.setColumnWidth(self.CURVE_COL, 80)
- self.main_ui.explo_tablewidget_variables.setColumnWidth(self.COLOR_COL, 80)
- self.main_ui.explo_tablewidget_variables.setColumnWidth(self.DELETE_COL, 80)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.CURSOR_COL, 60)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.COLOR_COL, 55)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.OFFSET_COL, 90)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.MULT_COL, 90)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.TIMESHIFT_COL, 95)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.VISIBLE_COL, 70)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.XORIG_COL, 80)
+ self.main_ui.explo_tablewidget_variables.setColumnWidth(self.YORIG_COL, 70)
self.__add_new_row_in_variable_table__()
def __add_new_row_in_variable_table__(self):
"""Add a new row in the calibration table. Initialize the cell's content and properties."""
- row_id = self.main_ui.explo_tablewidget_variables.rowCount()
- self.main_ui.explo_tablewidget_variables.insertRow(row_id)
+ table = self.main_ui.explo_tablewidget_variables
+ row_id = table.rowCount()
+ table.insertRow(row_id)
# Instruments
instrument_item = QComboBox()
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.INSTRUMENT_COL, instrument_item)
- self.main_ui.explo_tablewidget_variables.cellWidget(row_id, self.INSTRUMENT_COL).currentTextChanged.connect(
+ table.setCellWidget(row_id, self.INSTRUMENT_COL, instrument_item)
+ table.cellWidget(row_id, self.INSTRUMENT_COL).currentTextChanged.connect(
lambda text, row_id=row_id: self.__update_variables_combobox__(combobox_text=text, row=row_id))
# Variables
variable_item = QComboBox()
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.VARIABLE_COL, variable_item)
- self.main_ui.explo_tablewidget_variables.cellWidget(row_id, self.VARIABLE_COL).currentTextChanged.connect(
- lambda text, row_id=row_id: self.__apply_new_variable__(variable_name=text, row=row_id))
+ table.setCellWidget(row_id, self.VARIABLE_COL, variable_item)
+ table.cellWidget(row_id, self.VARIABLE_COL).currentTextChanged.connect(
+ lambda text, row_id=row_id: self.__apply_variable_change__(row_id=row_id, variable_name=text))
+
+ # Step Curve
+ curve_item = QCheckBox()
+ table.setCellWidget(row_id, self.CURVE_COL, curve_item)
# Vertical line
vline_item = QCheckBox()
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.VLINE_COL, vline_item)
+ table.setCellWidget(row_id, self.VLINE_COL, vline_item)
# Cursor
cursor_item = QCheckBox()
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.CURSOR_COL, cursor_item)
-
- # Curve
- curve_item = QCheckBox()
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.CURVE_COL, curve_item)
+ table.setCellWidget(row_id, self.CURSOR_COL, cursor_item)
# Color
color_item = QTableWidgetItem()
- self.main_ui.explo_tablewidget_variables.setItem(row_id, self.COLOR_COL, color_item)
-
- # Delete
- delete_item = QPushButton("Delete")
- self.main_ui.explo_tablewidget_variables.setCellWidget(row_id, self.DELETE_COL, delete_item)
-
- def __update_instruments_comboboxes__(self):
- instrument_combobox = self.main_ui.explo_tablewidget_variables.cellWidget(0, self.INSTRUMENT_COL) # TODO: ALL rows!!
+ color_item.setBackground(self.DEFAULT_COLORS[row_id % len(self.DEFAULT_COLORS)])
+ table.setItem(row_id, self.COLOR_COL, color_item)
+
+ # Offset
+ offset_item = QDoubleSpinBox()
+ offset_item.setMinimum(-1000.0)
+ table.setCellWidget(row_id, self.OFFSET_COL, offset_item)
+ table.cellWidget(row_id, self.OFFSET_COL).valueChanged.connect(
+ lambda value, row_id=row_id: self.__apply_variable_change__(row_id=row_id))
+
+ # Multiplicative factor
+ mult_item = QDoubleSpinBox()
+ mult_item.setValue(1.0)
+ mult_item.setMinimum(-1000.0)
+ table.setCellWidget(row_id, self.MULT_COL, mult_item)
+ table.cellWidget(row_id, self.MULT_COL).valueChanged.connect(
+ lambda value, row_id=row_id: self.__apply_variable_change__(row_id=row_id))
+
+ # Time shift
+ timeshift_item = QDoubleSpinBox()
+ timeshift_item.setMinimum(-1000.0)
+ table.setCellWidget(row_id, self.TIMESHIFT_COL, timeshift_item)
+ table.cellWidget(row_id, self.TIMESHIFT_COL).valueChanged.connect(
+ lambda value, row_id=row_id: self.__apply_variable_change__(row_id=row_id))
+
+ # Visible
+ visible_item = QCheckBox()
+ visible_item.setChecked(True)
+ table.setCellWidget(row_id, self.VISIBLE_COL, visible_item)
+
+ # X original
+ xorig_item = QTableWidgetItem()
+ table.setItem(row_id, self.XORIG_COL, xorig_item)
+
+ # Y original
+ yorig_item = QTableWidgetItem()
+ table.setItem(row_id, self.YORIG_COL, yorig_item)
+
+ self.__update_instruments_combobox__(row_id)
+
+ def __update_instruments_combobox__(self, row_id: int):
+ instrument_combobox = self.main_ui.explo_tablewidget_variables.cellWidget(row_id, self.INSTRUMENT_COL)
instrument_combobox.clear()
dataset_dir = self.main_ui.explo_combobox_dataset.currentText()
+ if dataset_dir == "":
+ return
dataset = self.explo_prvd.datasets[dataset_dir]
instrument_logs = dataset.instlogs
@@ -123,23 +190,74 @@ class ExploUim:
for variable_name in variable_names:
variables_combobox.addItem(variable_name)
- def __apply_new_variable__(self, variable_name: str, row: int):
- # The variable combobox is cleard before variables of the newly-selected instrument are written.
- # This function is called on variable-combobox edition, thus it is also called on combobox clear.
- # So in this case, do nothing, wait for the call related to the filling of the combobox
- if variable_name == "":
+ def __change_color__(self, row: int, column: int):
+ if column != self.COLOR_COL:
return
+ color = QColorDialog.getColor()
+ self.main_ui.explo_tablewidget_variables.item(row, self.COLOR_COL).setBackground(color)
+
+ def __get_row_dataframe__(self, row_id: int) -> pd.DataFrame:
+ table = self.main_ui.explo_tablewidget_variables
+
# Get instrument log
- instrument_log_name = self.main_ui.explo_tablewidget_variables.cellWidget(row, self.INSTRUMENT_COL).currentText()
+ instrument_log_name = table.cellWidget(row_id, self.INSTRUMENT_COL).currentText()
instrument_log = self.current_dataset.instlogs[instrument_log_name]
+ # Get variable name
+ variable_name = table.cellWidget(row_id, self.VARIABLE_COL).currentText()
+
timeseries = instrument_log.get_timeseries(variable_name)
+
+ return timeseries
+
+ def __apply_variable_change__(self, row_id: int, variable_name: str = None) -> None:
+ """Read the information related to the row_id and call plot function
+
+ Parameters
+ ----------
+ row_id
+ variable_name
+ """
+ # The variable combobox is cleared before variables of the newly-selected instrument are written.
+ # This function is called on variable-combobox edition, thus it is also called on combobox clear.
+ # So in this case, do nothing, wait for the call related to the filling of the combobox
+ if variable_name == "":
+ return
+
+ table = self.main_ui.explo_tablewidget_variables
+
+ timeseries = self.__get_row_dataframe__(row_id)
+
+ # Get color
+ color = table.item(row_id, self.COLOR_COL).background().color()
+
+ # Get position adjustment variables (offset, multiplicative factor and time shift)
+ offset = table.cellWidget(row_id, self.OFFSET_COL).value()
+ mult = table.cellWidget(row_id, self.MULT_COL).value()
+ timeshift = table.cellWidget(row_id, self.TIMESHIFT_COL).value()
+
+
try:
- self.__update_plot__(timeseries, variable_name)
+ self.__update_plot__(timeseries, row_id, color, offset, mult, timeshift)
except TypeError:
print("Cannot plot that.")
+ def __update_xy_original__(self, instant: datetime.datetime):
+ table = self.main_ui.explo_tablewidget_variables
+ for row_id in range(table.rowCount()):
+ # Get X orig (original non-shifted datetime value)
+ timeshift_sec = table.cellWidget(row_id, self.TIMESHIFT_COL).value()
+ x_orig = instant - datetime.timedelta(seconds=timeshift_sec)
+ table.item(row_id, self.XORIG_COL).setText(x_orig.strftime("%H:%M:%S.%f")[:-5])
+
+ # Get Y orig (original non-shifted variable value)
+ df = self.__get_row_dataframe__(row_id).copy()
+ y_orig = df[df["datetime"] <= instant].iloc[-1]["value"]
+ if isinstance(y_orig, float):
+ y_orig = "{:.4f}".format(y_orig)
+ table.item(row_id, self.YORIG_COL).setText(y_orig)
+
####################################################################
# Plot
@@ -147,25 +265,61 @@ class ExploUim:
self.plot_item = pg.PlotItem(axisItems={'bottom': utils.TimeAxisItem(orientation='bottom')})
- self.curve = pg.PlotCurveItem()
- self.plot_item.addItem(self.curve)
+ self.step_curves = dict()
self.main_ui.explo_graphicsview_top.setCentralItem(self.plot_item)
self.lines_items = []
- def __update_plot__(self, timeseries: pd.DataFrame, variable_name: str) -> None:
+ # Vertical line following the cursor
+ self.cursor_vline = pg.InfiniteLine(angle=90, movable=False)
+ self.plot_item.addItem(self.cursor_vline, ignoreBounds=True)
+
+ def __update_plot__(self, timeseries: pd.DataFrame, row_id: int, color: QColor,
+ offset: float, mult: float, timeshift_sec: float) -> None:
try:
- timeseries[variable_name] = timeseries[variable_name].astype(float)
+ timeseries["value"] = timeseries["value"].astype(float)
except ValueError:
print("Value is not convertible to float")
+ return
+
+ # Get the to-be-modified curve
+ if row_id in self.step_curves:
+ step_curve = self.step_curves[row_id]
else:
- last_datetime = self.current_dataset.last_data_datetime + datetime.timedelta(seconds=1)
- timeseries['datetime'].at[max(timeseries.index)+1] = last_datetime
- self.curve.setData(x=utils.pd_time_to_epoch_ms(timeseries['datetime']),
- y=list(timeseries[variable_name]),
- pen=(200, 200, 200),
- stepMode=True,
- symbol="o",
- symbolBrush=(255, 0, 0),
- symbolPen='w')
+ step_curve = pg.PlotCurveItem()
+ self.plot_item.addItem(step_curve)
+ step_curve.scene().sigMouseClicked.connect(lambda event: self.__mouse_clicked__(event))
+ self.step_curves[row_id] = step_curve
+
+ # As it is a _step_ curve, add a last datetime point to determine the end of the last step. This is the datatime
+ # of the last available data of the dataset, plus one second.
+ last_datetime = self.current_dataset.last_data_datetime + datetime.timedelta(seconds=1)
+ x_values = timeseries['datetime'].copy()
+ x_values = x_values.append(pd.Series([last_datetime]))
+
+ # Apply time shift
+ x_values = x_values + datetime.timedelta(seconds=timeshift_sec)
+
+ y_values = list(timeseries["value"])
+
+ # Apply multiplicative factor
+ y_values = [y * mult for y in y_values]
+
+ # Apply Y-axis offset
+ y_values = [y + offset for y in y_values]
+
+ # Set data to the plot
+ step_curve.setData(x=utils.pd_time_to_epoch_ms(x_values),
+ y=y_values,
+ pen=color,
+ stepMode=True)
+
+ def __mouse_clicked__(self, event: MouseClickEvent):
+ pos = event.scenePos()
+ mousePoint = self.step_curves[0].getViewBox().mapSceneToView(pos)
+ instant = datetime.datetime.fromtimestamp(mousePoint.x(), tz=datetime.timezone.utc)
+ # instant = datetime.datetime.fromtimestamp(mousePoint.x())
+ self.__update_xy_original__(instant)
+ self.cursor_vline.setPos(mousePoint.x())
+ # print("x="+instant.strftime("%H:%M:%S"))
--
GitLab