Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
C
CFA Analysis
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
JOSSOUD Olivier
CFA Analysis
Commits
e1f8beb3
Commit
e1f8beb3
authored
2 years ago
by
JOSSOUD Olivier
Browse files
Options
Downloads
Patches
Plain Diff
Analyse. Use in-app calibration editor.
parent
53b84b35
No related branches found
Branches containing commit
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
.idea/watcherTasks.xml
+1
-1
1 addition, 1 deletion
.idea/watcherTasks.xml
src/controller/analysecontroller.py
+119
-66
119 additions, 66 deletions
src/controller/analysecontroller.py
src/uim/analyseuim.py
+67
-7
67 additions, 7 deletions
src/uim/analyseuim.py
with
187 additions
and
74 deletions
.idea/watcherTasks.xml
+
1
−
1
View file @
e1f8beb3
...
@@ -2,7 +2,7 @@
...
@@ -2,7 +2,7 @@
<project
version=
"4"
>
<project
version=
"4"
>
<component
name=
"ProjectTasksOptions"
>
<component
name=
"ProjectTasksOptions"
>
<TaskOptions
isEnabled=
"true"
>
<TaskOptions
isEnabled=
"true"
>
<option
name=
"arguments"
value=
"$FilePath$"
/>
<option
name=
"arguments"
value=
"$FilePath$
-l 120
"
/>
<option
name=
"checkSyntaxErrors"
value=
"true"
/>
<option
name=
"checkSyntaxErrors"
value=
"true"
/>
<option
name=
"description"
/>
<option
name=
"description"
/>
<option
name=
"exitCodeBehavior"
value=
"ERROR"
/>
<option
name=
"exitCodeBehavior"
value=
"ERROR"
/>
...
...
This diff is collapsed.
Click to expand it.
src/controller/analysecontroller.py
+
119
−
66
View file @
e1f8beb3
...
@@ -10,7 +10,8 @@ import cfatools.processor.flow as flow
...
@@ -10,7 +10,8 @@ import cfatools.processor.flow as flow
import
cfatools.processor.collector
as
collector
import
cfatools.processor.collector
as
collector
import
cfatools.processor.encoder
as
encoder
import
cfatools.processor.encoder
as
encoder
from
cfatools.logreader.dataset
import
DatasetReader
from
cfatools.logreader.dataset
import
DatasetReader
matplotlib
.
use
(
'
TkAgg
'
)
matplotlib
.
use
(
"
TkAgg
"
)
class
AnalyseController
(
QObject
):
class
AnalyseController
(
QObject
):
...
@@ -35,6 +36,7 @@ class AnalyseController(QObject):
...
@@ -35,6 +36,7 @@ class AnalyseController(QObject):
self
.
flask_df
=
pd
.
DataFrame
()
self
.
flask_df
=
pd
.
DataFrame
()
self
.
flask_df_filepath
=
""
self
.
flask_df_filepath
=
""
self
.
pic_conduc_height_filepath
=
""
self
.
pic_conduc_height_filepath
=
""
self
.
calib_filepath
=
""
self
.
calib_df
=
pd
.
DataFrame
()
self
.
calib_df
=
pd
.
DataFrame
()
self
.
pic_conduct_calibrated_filepath
=
""
self
.
pic_conduct_calibrated_filepath
=
""
self
.
flask_calibrated_filepath
=
""
self
.
flask_calibrated_filepath
=
""
...
@@ -45,16 +47,20 @@ class AnalyseController(QObject):
...
@@ -45,16 +47,20 @@ class AnalyseController(QObject):
valid
,
error_msg
=
DatasetReader
.
dataset_name_is_valid
(
dataset_name
)
valid
,
error_msg
=
DatasetReader
.
dataset_name_is_valid
(
dataset_name
)
if
not
valid
:
if
not
valid
:
raise
ValueError
(
error_msg
)
raise
ValueError
(
error_msg
)
self
.
current_dataset
=
DatasetReader
(
self
.
current_dataset
=
DatasetReader
(
base_path
=
root_directory
,
dataset
=
dataset_name
)
base_path
=
root_directory
,
dataset
=
dataset_name
)
self
.
sig_simple_step_finished
.
emit
(
"
Dataset [
"
+
self
.
current_dataset
.
dataset_name
+
"
] loaded.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Dataset [
"
+
self
.
current_dataset
.
dataset_name
+
"
] loaded.
"
)
return
self
.
current_dataset
return
self
.
current_dataset
def
analyse
(
self
,
override_arrival_pkl
:
bool
=
False
):
def
analyse
(
self
,
override_arrival_pkl
:
bool
=
False
):
dataset
=
self
.
current_dataset
dataset
=
self
.
current_dataset
self
.
encoder_df
,
self
.
iceblock_df
,
self
.
conduct_df
,
self
.
pump_df
,
self
.
picarro_df
,
self
.
collector_df
=
\
(
flow
.
get_datasets_data
(
dataset
)
self
.
encoder_df
,
self
.
iceblock_df
,
self
.
conduct_df
,
self
.
pump_df
,
self
.
picarro_df
,
self
.
collector_df
,
)
=
flow
.
get_datasets_data
(
dataset
)
self
.
__apply_special_corrections__
()
self
.
__apply_special_corrections__
()
...
@@ -69,8 +75,9 @@ class AnalyseController(QObject):
...
@@ -69,8 +75,9 @@ class AnalyseController(QObject):
################################################################################################################
################################################################################################################
# Tubing volumes
# Tubing volumes
self
.
volumes_dict
=
flow
.
get_tubing_volume_dict
(
"
../config/tubing_volumes.csv
"
,
self
.
volumes_dict
=
flow
.
get_tubing_volume_dict
(
max_datetime
=
self
.
encoder_df
.
index
[
0
])
"
../config/tubing_volumes.csv
"
,
max_datetime
=
self
.
encoder_df
.
index
[
0
]
)
self
.
sig_action_step_finished
.
emit
(
"
Volume table loaded.
"
,
"
View
"
,
"
view_volume_dict
"
)
self
.
sig_action_step_finished
.
emit
(
"
Volume table loaded.
"
,
"
View
"
,
"
view_volume_dict
"
)
################################################################################################################
################################################################################################################
...
@@ -80,8 +87,14 @@ class AnalyseController(QObject):
...
@@ -80,8 +87,14 @@ class AnalyseController(QObject):
if
os
.
path
.
exists
(
pkl_filepath
):
if
os
.
path
.
exists
(
pkl_filepath
):
os
.
remove
(
pkl_filepath
)
os
.
remove
(
pkl_filepath
)
self
.
sig_simple_step_finished
.
emit
(
"
Previous arrival_nearest.pkl file deleted.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Previous arrival_nearest.pkl file deleted.
"
)
self
.
arrival_df_raw
=
flow
.
get_arrival_df
(
self
.
encoder_df
,
self
.
pump_df
,
self
.
volumes_dict
,
self
.
arrival_df_raw
=
flow
.
get_arrival_df
(
parallel
=
False
,
dataset
=
dataset
,
pkl_name
=
"
arrival_nearest.pkl
"
)
self
.
encoder_df
,
self
.
pump_df
,
self
.
volumes_dict
,
parallel
=
False
,
dataset
=
dataset
,
pkl_name
=
"
arrival_nearest.pkl
"
,
)
if
override_arrival_pkl
:
if
override_arrival_pkl
:
self
.
sig_action_step_finished
.
emit
(
"
Arrival_df computed.
"
,
"
View
"
,
"
view_arrival_df
"
)
self
.
sig_action_step_finished
.
emit
(
"
Arrival_df computed.
"
,
"
View
"
,
"
view_arrival_df
"
)
else
:
else
:
...
@@ -91,14 +104,20 @@ class AnalyseController(QObject):
...
@@ -91,14 +104,20 @@ class AnalyseController(QObject):
self
.
arrival_df
=
flow
.
add_iceblock_info
(
self
.
arrival_df
,
self
.
iceblock_df
)
self
.
arrival_df
=
flow
.
add_iceblock_info
(
self
.
arrival_df
,
self
.
iceblock_df
)
self
.
arrival_df
=
flow
.
add_melted_height
(
self
.
arrival_df
,
self
.
encoder_df
,
self
.
iceblock_df
)
self
.
arrival_df
=
flow
.
add_melted_height
(
self
.
arrival_df
,
self
.
encoder_df
,
self
.
iceblock_df
)
self
.
arrival_df
=
flow
.
add_flask_info
(
self
.
arrival_df
,
self
.
collector_df
)
self
.
arrival_df
=
flow
.
add_flask_info
(
self
.
arrival_df
,
self
.
collector_df
)
self
.
sig_action_step_finished
.
emit
(
"
Arrival_df augmented with iceblock, encoder and flask info.
"
,
self
.
sig_action_step_finished
.
emit
(
"
View
"
,
"
view_arrival_df
"
)
"
Arrival_df augmented with iceblock, encoder and flask info.
"
,
"
View
"
,
"
view_arrival_df
"
,
)
# Save as CSV
# Save as CSV
self
.
arrival_filepath
=
os
.
path
.
join
(
processed_dir
,
dataset
.
dataset_name
+
"
_arrival_df.csv
"
)
self
.
arrival_filepath
=
os
.
path
.
join
(
processed_dir
,
dataset
.
dataset_name
+
"
_arrival_df.csv
"
)
self
.
arrival_df
.
to_csv
(
self
.
arrival_filepath
,
date_format
=
"
%Y-%m-%d %H:%M:%S
"
)
self
.
arrival_df
.
to_csv
(
self
.
arrival_filepath
,
date_format
=
"
%Y-%m-%d %H:%M:%S
"
)
self
.
sig_action_step_finished
.
emit
(
"
Arrival_df saved as CSV in [
"
+
self
.
arrival_filepath
+
"
]
"
,
self
.
sig_action_step_finished
.
emit
(
"
Open
"
,
"
open_arrival_csv
"
)
"
Arrival_df saved as CSV in [
"
+
self
.
arrival_filepath
+
"
]
"
,
"
Open
"
,
"
open_arrival_csv
"
,
)
################################################################################################################
################################################################################################################
# Conducti rescaled by melting time
# Conducti rescaled by melting time
...
@@ -109,12 +128,13 @@ class AnalyseController(QObject):
...
@@ -109,12 +128,13 @@ class AnalyseController(QObject):
# Picarro rescaled by melting time
# Picarro rescaled by melting time
if
not
self
.
picarro_df
.
index
.
is_monotonic_increasing
:
if
not
self
.
picarro_df
.
index
.
is_monotonic_increasing
:
self
.
picarro_df
.
reset_index
(
inplace
=
True
)
self
.
picarro_df
.
reset_index
(
inplace
=
True
)
self
.
picarro_df
[
'
diff_time
'
]
=
self
.
picarro_df
[
"
datetime
"
]
-
self
.
picarro_df
[
"
datetime
"
].
shift
(
1
)
self
.
picarro_df
[
"
diff_time
"
]
=
self
.
picarro_df
[
"
datetime
"
]
-
self
.
picarro_df
[
"
datetime
"
].
shift
(
1
)
self
.
picarro_df
=
self
.
picarro_df
.
loc
[
self
.
picarro_df
=
self
.
picarro_df
.
loc
[
self
.
picarro_df
.
index
>
self
.
picarro_df
.
loc
[
self
.
picarro_df
[
"
diff_time
"
]
self
.
picarro_df
.
index
.
dt
.
total_seconds
()
<
0
].
index
[
0
]]
>
self
.
picarro_df
.
loc
[
self
.
picarro_df
[
"
diff_time
"
].
dt
.
total_seconds
()
<
0
].
index
[
0
]
]
self
.
picarro_df
=
self
.
picarro_df
.
set_index
(
"
datetime
"
)
self
.
picarro_df
=
self
.
picarro_df
.
set_index
(
"
datetime
"
)
self
.
picarro_df
=
self
.
picarro_df
.
drop
(
columns
=
{
'
diff_time
'
})
self
.
picarro_df
=
self
.
picarro_df
.
drop
(
columns
=
{
"
diff_time
"
})
picarro_rescaled_df
=
flow
.
get_picarro_by_melt_time
(
self
.
arrival_df
,
self
.
picarro_df
,
export_in_dataset
=
dataset
)
picarro_rescaled_df
=
flow
.
get_picarro_by_melt_time
(
self
.
arrival_df
,
self
.
picarro_df
,
export_in_dataset
=
dataset
)
self
.
sig_simple_step_finished
.
emit
(
"
Picarro-R(escaled) created.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Picarro-R(escaled) created.
"
)
...
@@ -126,8 +146,11 @@ class AnalyseController(QObject):
...
@@ -126,8 +146,11 @@ class AnalyseController(QObject):
# Save as CSV
# Save as CSV
self
.
flask_df_filepath
=
os
.
path
.
join
(
processed_dir
,
dataset
.
dataset_name
+
"
_flask_df.csv
"
)
self
.
flask_df_filepath
=
os
.
path
.
join
(
processed_dir
,
dataset
.
dataset_name
+
"
_flask_df.csv
"
)
self
.
flask_df
.
to_csv
(
self
.
flask_df_filepath
,
index
=
False
)
self
.
flask_df
.
to_csv
(
self
.
flask_df_filepath
,
index
=
False
)
self
.
sig_action_step_finished
.
emit
(
"
flask_df saved as CSV in [
"
+
self
.
arrival_filepath
+
"
]
"
,
self
.
sig_action_step_finished
.
emit
(
"
Open
"
,
"
open_flask_csv
"
)
"
flask_df saved as CSV in [
"
+
self
.
arrival_filepath
+
"
]
"
,
"
Open
"
,
"
open_flask_csv
"
,
)
# Plot mm per flask
# Plot mm per flask
# if not all(self.flask_df["flask"] == 0):
# if not all(self.flask_df["flask"] == 0):
...
@@ -149,7 +172,9 @@ class AnalyseController(QObject):
...
@@ -149,7 +172,9 @@ class AnalyseController(QObject):
pic_conduc_height_df
.
to_csv
(
self
.
pic_conduc_height_filepath
)
pic_conduc_height_df
.
to_csv
(
self
.
pic_conduc_height_filepath
)
self
.
sig_action_step_finished
.
emit
(
self
.
sig_action_step_finished
.
emit
(
"
pic_conduc_height_df saved as CSV in [
"
+
self
.
pic_conduc_height_filepath
+
"
]
"
,
"
pic_conduc_height_df saved as CSV in [
"
+
self
.
pic_conduc_height_filepath
+
"
]
"
,
"
Open
"
,
"
open_pic_conduc_height_df
"
)
"
Open
"
,
"
open_pic_conduc_height_df
"
,
)
################################################################################################################
################################################################################################################
# Picarro
# Picarro
...
@@ -176,25 +201,34 @@ class AnalyseController(QObject):
...
@@ -176,25 +201,34 @@ class AnalyseController(QObject):
calibrated_dir
=
os
.
path
.
join
(
dataset
.
dataset_path
,
"
calibrated
"
)
calibrated_dir
=
os
.
path
.
join
(
dataset
.
dataset_path
,
"
calibrated
"
)
if
not
os
.
path
.
exists
(
calibrated_dir
):
if
not
os
.
path
.
exists
(
calibrated_dir
):
os
.
mkdir
(
calibrated_dir
)
os
.
mkdir
(
calibrated_dir
)
calib_filepath
=
os
.
path
.
join
(
calibrated_dir
,
dataset
.
dataset_name
+
"
_
"
+
calib_name
+
"
.csv
"
)
self
.
calib_filepath
=
os
.
path
.
join
(
calibrated_dir
,
dataset
.
dataset_name
+
"
_
"
+
calib_name
+
"
.csv
"
)
if
not
os
.
path
.
exists
(
calib_filepath
):
if
not
os
.
path
.
exists
(
self
.
calib_filepath
):
template_calib_df
=
self
.
iceblock_df
[[
"
id
"
,
"
name
"
,
"
initial_height
"
]].
copy
()
template_calib_df
=
self
.
iceblock_df
[[
"
id
"
,
"
name
"
,
"
initial_height
"
]].
copy
()
template_calib_df
[
"
start
"
]
=
0
template_calib_df
[
"
start
"
]
=
0
template_calib_df
=
template_calib_df
\
template_calib_df
=
(
.
melt
([
"
id
"
,
"
name
"
])
\
template_calib_df
.
melt
([
"
id
"
,
"
name
"
]).
sort_values
([
"
id
"
,
"
value
"
]).
drop
(
columns
=
[
"
id
"
,
"
variable
"
])
.
sort_values
([
"
id
"
,
"
value
"
])
\
)
.
drop
(
columns
=
[
"
id
"
,
"
variable
"
])
template_calib_df
=
template_calib_df
.
rename
(
columns
=
{
"
name
"
:
"
icbk_name
"
,
"
value
"
:
"
melted_height
"
})
template_calib_df
=
template_calib_df
.
rename
(
columns
=
{
"
name
"
:
"
icbk_name
"
,
template_calib_df
[
"
initial_height
"
:
"
melted_height
"
})
[
template_calib_df
[[
"
depth
"
,
"
depth
"
,
"
Delta_18_16_slope
"
,
"
Delta_18_16_intercept
"
,
"
Delta_18_16_slope
"
,
"
Delta_D_H_slope
"
,
"
Delta_D_H_intercept
"
]]
=
[
np
.
NaN
,
np
.
NaN
,
np
.
NaN
,
np
.
NaN
,
np
.
NaN
]
"
Delta_18_16_intercept
"
,
template_calib_df
.
to_csv
(
calib_filepath
,
index
=
False
,
float_format
=
'
%.2f
'
)
"
Delta_D_H_slope
"
,
self
.
calib_df
=
pd
.
read_csv
(
calib_filepath
,
sep
=
"
,
"
)
"
Delta_D_H_intercept
"
,
]
if
len
(
self
.
calib_df
.
index
)
<
1
:
]
=
[
np
.
NaN
,
np
.
NaN
,
np
.
NaN
,
np
.
NaN
,
np
.
NaN
]
self
.
sig_simple_step_finished
.
emit
(
"
Calibration file is empty: [
"
+
calib_filepath
+
"
]
"
)
template_calib_df
.
to_csv
(
self
.
calib_filepath
,
index
=
False
,
float_format
=
"
%.2f
"
,
sep
=
"
;
"
)
self
.
calib_df
=
pd
.
read_csv
(
self
.
calib_filepath
,
sep
=
"
;
"
)
if
all
(
self
.
calib_df
[
"
depth
"
].
isna
()):
self
.
sig_action_step_finished
.
emit
(
"
Calibration file is empty: [
"
+
self
.
calib_filepath
+
"
]
"
,
"
Edit
"
,
"
edit_calib_df
"
,
)
self
.
sig_simple_step_finished
.
emit
(
"
Abort calibration.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Abort calibration.
"
)
return
else
:
else
:
self
.
sig_action_step_finished
.
emit
(
"
Calibration file loaded.
"
,
"
View
"
,
"
view_calib_df
"
)
self
.
sig_action_step_finished
.
emit
(
"
Calibration file loaded.
"
,
"
View
"
,
"
view_calib_df
"
)
...
@@ -203,57 +237,68 @@ class AnalyseController(QObject):
...
@@ -203,57 +237,68 @@ class AnalyseController(QObject):
# Absolute depth calibration
# Absolute depth calibration
for
icbk_name
in
self
.
calib_df
[
"
icbk_name
"
].
unique
():
for
icbk_name
in
self
.
calib_df
[
"
icbk_name
"
].
unique
():
icbk_calib_df
=
self
.
calib_df
.
loc
[
self
.
calib_df
[
"
icbk_name
"
]
==
icbk_name
]
icbk_calib_df
=
self
.
calib_df
.
loc
[
self
.
calib_df
[
"
icbk_name
"
]
==
icbk_name
]
depth_interp_func
=
interpolate
.
interp1d
(
x
=
icbk_calib_df
[
"
melted_height
"
],
depth_interp_func
=
interpolate
.
interp1d
(
x
=
icbk_calib_df
[
"
melted_height
"
],
y
=
icbk_calib_df
[
"
depth
"
])
y
=
icbk_calib_df
[
"
depth
"
])
try
:
try
:
pic_conduc_height_df
.
loc
[
pic_conduc_height_df
[
"
icbk_name
"
]
==
icbk_name
,
"
depth
"
]
\
pic_conduc_height_df
.
loc
[
pic_conduc_height_df
[
"
icbk_name
"
]
==
icbk_name
,
"
depth
"
]
=
depth_interp_func
(
=
depth_interp_func
(
pic_conduc_height_df
pic_conduc_height_df
.
loc
[
pic_conduc_height_df
[
"
icbk_name
"
]
==
icbk_name
][
"
melted_height_icbk
"
]
.
loc
[
pic_conduc_height_df
[
"
icbk_name
"
]
==
icbk_name
][
"
melted_height_icbk
"
]
)
)
except
:
except
:
print
(
icbk_name
)
print
(
icbk_name
)
# Picarro calibration
# Picarro calibration
if
len
(
self
.
calib_df
.
groupby
([
"
Delta_18_16_slope
"
,
"
Delta_18_16_intercept
"
,
if
(
"
Delta_D_H_slope
"
,
"
Delta_D_H_intercept
"
]).
count
())
>
1
:
len
(
self
.
calib_df
.
groupby
(
[
"
Delta_18_16_slope
"
,
"
Delta_18_16_intercept
"
,
"
Delta_D_H_slope
"
,
"
Delta_D_H_intercept
"
]
).
count
()
)
>
1
):
raise
ValueError
(
"
More than one calibration set for isotope is not yet supported
"
)
raise
ValueError
(
"
More than one calibration set for isotope is not yet supported
"
)
isotopic_calib_df
=
self
.
calib_df
.
iloc
[
0
]
isotopic_calib_df
=
self
.
calib_df
.
iloc
[
0
]
calib_pic_conduc_height_df
=
pic_conduc_height_df
.
copy
()
calib_pic_conduc_height_df
=
pic_conduc_height_df
.
copy
()
calib_pic_conduc_height_df
[
"
Delta_18_16_calib
"
]
=
\
calib_pic_conduc_height_df
[
"
Delta_18_16_calib
"
]
=
(
calib_pic_conduc_height_df
[
"
Delta_18_16
"
]
*
isotopic_calib_df
.
Delta_18_16_slope
\
calib_pic_conduc_height_df
[
"
Delta_18_16
"
]
*
isotopic_calib_df
.
Delta_18_16_slope
+
isotopic_calib_df
.
Delta_18_16_intercept
+
isotopic_calib_df
.
Delta_18_16_intercept
calib_pic_conduc_height_df
[
"
Delta_D_H_calib
"
]
=
\
)
calib_pic_conduc_height_df
[
"
Delta_D_H
"
]
*
isotopic_calib_df
.
Delta_D_H_slope
\
calib_pic_conduc_height_df
[
"
Delta_D_H_calib
"
]
=
(
calib_pic_conduc_height_df
[
"
Delta_D_H
"
]
*
isotopic_calib_df
.
Delta_D_H_slope
+
isotopic_calib_df
.
Delta_D_H_intercept
+
isotopic_calib_df
.
Delta_D_H_intercept
)
# Save as CSV
# Save as CSV
self
.
pic_conduct_calibrated_filepath
=
os
.
path
.
join
(
self
.
pic_conduct_calibrated_filepath
=
os
.
path
.
join
(
calibrated_dir
,
calibrated_dir
,
dataset
.
dataset_name
+
"
_pic_conduct_calibrated_
"
+
calib_name
+
"
.csv
"
dataset
.
dataset_name
+
"
_pic_conduct_calibrated_
"
+
calib_name
+
"
.csv
"
)
)
calib_pic_conduc_height_df
.
to_csv
(
self
.
pic_conduct_calibrated_filepath
,
float_format
=
'
%.3f
'
)
calib_pic_conduc_height_df
.
to_csv
(
self
.
pic_conduct_calibrated_filepath
,
float_format
=
"
%.3f
"
)
self
.
sig_action_step_finished
.
emit
(
self
.
sig_action_step_finished
.
emit
(
"
Calibrated Picarro & Conducti saved as CSV in [
"
+
self
.
pic_conduct_calibrated_filepath
+
"
]
"
,
"
Calibrated Picarro & Conducti saved as CSV in [
"
+
self
.
pic_conduct_calibrated_filepath
+
"
]
"
,
"
Open
"
,
"
open_calib_pic_conduc_height_df
"
)
"
Open
"
,
"
open_calib_pic_conduc_height_df
"
,
)
################################################################################################################
################################################################################################################
# Calibrate flasks
# Calibrate flasks
calib_flask_df
=
self
.
flask_df
.
copy
()
calib_flask_df
=
self
.
flask_df
.
copy
()
for
icbk_name
in
self
.
calib_df
[
"
icbk_name
"
].
unique
():
for
icbk_name
in
self
.
calib_df
[
"
icbk_name
"
].
unique
():
icbk_calib_df
=
self
.
calib_df
.
loc
[
self
.
calib_df
[
"
icbk_name
"
]
==
icbk_name
]
icbk_calib_df
=
self
.
calib_df
.
loc
[
self
.
calib_df
[
"
icbk_name
"
]
==
icbk_name
]
depth_interp_func
=
interpolate
.
interp1d
(
x
=
icbk_calib_df
[
"
melted_height
"
],
depth_interp_func
=
interpolate
.
interp1d
(
x
=
icbk_calib_df
[
"
melted_height
"
],
y
=
icbk_calib_df
[
"
depth
"
])
y
=
icbk_calib_df
[
"
depth
"
])
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
,
"
min_depth
"
]
=
depth_interp_func
(
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
,
"
min_depth
"
]
\
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
][
"
min
"
]
=
depth_interp_func
(
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
][
"
min
"
])
)
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
,
"
max_depth
"
]
\
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
,
"
max_depth
"
]
=
depth_interp_func
(
=
depth_interp_func
(
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
][
"
max
"
])
calib_flask_df
.
loc
[
calib_flask_df
[
"
icbk_name
"
]
==
icbk_name
][
"
max
"
]
)
calib_flask_df
[
"
diff_depth
"
]
=
calib_flask_df
[
"
max_depth
"
]
-
calib_flask_df
[
"
min_depth
"
]
calib_flask_df
[
"
diff_depth
"
]
=
calib_flask_df
[
"
max_depth
"
]
-
calib_flask_df
[
"
min_depth
"
]
self
.
flask_calibrated_filepath
=
os
.
path
.
join
(
calibrated_dir
,
self
.
flask_calibrated_filepath
=
os
.
path
.
join
(
dataset
.
dataset_name
+
"
_flask_calibrated_
"
+
calib_name
+
"
.csv
"
)
calibrated_dir
,
dataset
.
dataset_name
+
"
_flask_calibrated_
"
+
calib_name
+
"
.csv
"
calib_flask_df
.
to_csv
(
self
.
flask_calibrated_filepath
,
float_format
=
'
%.2f
'
,
index
=
False
)
)
calib_flask_df
.
to_csv
(
self
.
flask_calibrated_filepath
,
float_format
=
"
%.2f
"
,
index
=
False
)
self
.
sig_action_step_finished
.
emit
(
self
.
sig_action_step_finished
.
emit
(
"
Calibrated Flask saved as CSV in [
"
+
self
.
flask_calibrated_filepath
+
"
]
"
,
"
Calibrated Flask saved as CSV in [
"
+
self
.
flask_calibrated_filepath
+
"
]
"
,
"
Open
"
,
"
open_calib_flask_df
"
"
Open
"
,
"
open_calib_flask_df
"
)
)
def
__apply_special_corrections__
(
self
)
->
None
:
def
__apply_special_corrections__
(
self
)
->
None
:
"""
Some datasets require some manual correction before being processed, due to errors during the recording
"""
"""
Some datasets require some manual correction before being processed, due to errors during the recording
"""
...
@@ -274,7 +319,8 @@ class AnalyseController(QObject):
...
@@ -274,7 +319,8 @@ class AnalyseController(QObject):
self
.
encoder_df
,
self
.
encoder_df
,
old_peak_start
=
pd
.
Timestamp
(
"
2022-03-17 13:28:27.446578+00:00
"
),
old_peak_start
=
pd
.
Timestamp
(
"
2022-03-17 13:28:27.446578+00:00
"
),
old_peak_end
=
pd
.
Timestamp
(
"
2022-03-17 13:28:41.721330+00:00
"
),
old_peak_end
=
pd
.
Timestamp
(
"
2022-03-17 13:28:41.721330+00:00
"
),
shift
=
pd
.
Timedelta
(
seconds
=
60
))
shift
=
pd
.
Timedelta
(
seconds
=
60
),
)
self
.
sig_simple_step_finished
.
emit
(
"
Special correction applied on encoder_df.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Special correction applied on encoder_df.
"
)
if
self
.
current_dataset
.
dataset_name
==
"
20220329_ASUMA2016_23_12sq
"
:
if
self
.
current_dataset
.
dataset_name
==
"
20220329_ASUMA2016_23_12sq
"
:
...
@@ -282,9 +328,13 @@ class AnalyseController(QObject):
...
@@ -282,9 +328,13 @@ class AnalyseController(QObject):
self
.
encoder_df
,
self
.
encoder_df
,
old_peak_start
=
pd
.
Timestamp
(
"
2022-03-29 16:58:11.552587+00:00
"
),
old_peak_start
=
pd
.
Timestamp
(
"
2022-03-29 16:58:11.552587+00:00
"
),
old_peak_end
=
pd
.
Timestamp
(
"
2022-03-29 16:58:48.006613+00:00
"
),
old_peak_end
=
pd
.
Timestamp
(
"
2022-03-29 16:58:48.006613+00:00
"
),
shift
=
pd
.
Timedelta
(
seconds
=
80
))
shift
=
pd
.
Timedelta
(
seconds
=
80
),
)
self
.
sig_simple_step_finished
.
emit
(
"
Special correction applied on encoder_df.
"
)
self
.
sig_simple_step_finished
.
emit
(
"
Special correction applied on encoder_df.
"
)
def
update_calib_csv
(
self
,
calib_df
:
pd
.
DataFrame
):
calib_df
.
to_csv
(
self
.
calib_filepath
,
index
=
False
,
float_format
=
"
%.5f
"
,
sep
=
"
;
"
)
class
Worker
(
QObject
):
class
Worker
(
QObject
):
"""
Worker class used to move long
"
analyse
"
computations to another thread, to avoid GUI freezing.
"""
"""
Worker class used to move long
"
analyse
"
computations to another thread, to avoid GUI freezing.
"""
...
@@ -296,5 +346,8 @@ class Worker(QObject):
...
@@ -296,5 +346,8 @@ class Worker(QObject):
self
.
ctrl
=
controller
self
.
ctrl
=
controller
def
run
(
self
):
def
run
(
self
):
self
.
ctrl
.
analyse
()
try
:
self
.
ctrl
.
analyse
()
except
Exception
as
e
:
print
(
e
)
self
.
finished
.
emit
()
self
.
finished
.
emit
()
This diff is collapsed.
Click to expand it.
src/uim/analyseuim.py
+
67
−
7
View file @
e1f8beb3
...
@@ -15,7 +15,11 @@ class AnalyseUim(QObject):
...
@@ -15,7 +15,11 @@ class AnalyseUim(QObject):
"""
Class managing the Widget which are common for all Picarro calib sub-tabs.
"""
"""
Class managing the Widget which are common for all Picarro calib sub-tabs.
"""
def
__init__
(
def
__init__
(
self
,
ctrl
:
AnalyseController
,
main_ui
:
Ui_MainWindow
,
config
:
Config
,
main_window
self
,
ctrl
:
AnalyseController
,
main_ui
:
Ui_MainWindow
,
config
:
Config
,
main_window
,
):
):
super
(
AnalyseUim
,
self
).
__init__
()
super
(
AnalyseUim
,
self
).
__init__
()
self
.
main_ui
=
main_ui
self
.
main_ui
=
main_ui
...
@@ -25,7 +29,9 @@ class AnalyseUim(QObject):
...
@@ -25,7 +29,9 @@ class AnalyseUim(QObject):
self
.
__init_tableview__
()
self
.
__init_tableview__
()
self
.
main_ui
.
analyse_pushbutton_dataset_load
.
clicked
.
connect
(
self
.
__load_dataset__
)
self
.
main_ui
.
analyse_pushbutton_dataset_load
.
clicked
.
connect
(
self
.
__load_dataset__
)
self
.
main_ui
.
analyse_toolbutton_run
.
clicked
.
connect
(
self
.
__run__
)
self
.
main_ui
.
analyse_toolbutton_run
.
clicked
.
connect
(
self
.
__run__
)
self
.
ctrl
.
sig_simple_step_finished
.
connect
(
self
.
__display_simple_step__
)
self
.
ctrl
.
sig_simple_step_finished
.
connect
(
self
.
__display_simple_step__
)
self
.
ctrl
.
sig_action_step_finished
.
connect
(
self
.
__display_step_with_button__
)
self
.
ctrl
.
sig_action_step_finished
.
connect
(
self
.
__display_step_with_button__
)
...
@@ -42,7 +48,9 @@ class AnalyseUim(QObject):
...
@@ -42,7 +48,9 @@ class AnalyseUim(QObject):
utils
.
show_popup
(
str
(
e
),
"
Could not load data set!
"
)
utils
.
show_popup
(
str
(
e
),
"
Could not load data set!
"
)
return
return
self
.
main_ui
.
analyse_lineedit_dataset
.
setText
(
self
.
ctrl
.
current_dataset
.
dataset_name
)
self
.
main_ui
.
analyse_lineedit_dataset
.
setText
(
self
.
ctrl
.
current_dataset
.
dataset_name
)
def
__run__
(
self
):
def
__run__
(
self
):
"""
Run analyse, in another thread to prevent GUI freezing.
"""
"""
Run analyse, in another thread to prevent GUI freezing.
"""
...
@@ -54,7 +62,9 @@ class AnalyseUim(QObject):
...
@@ -54,7 +62,9 @@ class AnalyseUim(QObject):
self
.
worker
.
finished
.
connect
(
self
.
thread
.
quit
)
self
.
worker
.
finished
.
connect
(
self
.
thread
.
quit
)
self
.
worker
.
finished
.
connect
(
self
.
worker
.
deleteLater
)
self
.
worker
.
finished
.
connect
(
self
.
worker
.
deleteLater
)
self
.
thread
.
finished
.
connect
(
self
.
thread
.
deleteLater
)
self
.
thread
.
finished
.
connect
(
self
.
thread
.
deleteLater
)
self
.
worker
.
finished
.
connect
(
lambda
:
self
.
main_ui
.
analyse_toolbutton_run
.
setEnabled
(
True
))
self
.
worker
.
finished
.
connect
(
lambda
:
self
.
main_ui
.
analyse_toolbutton_run
.
setEnabled
(
True
)
)
self
.
thread
.
start
()
self
.
thread
.
start
()
def
__init_tableview__
(
self
):
def
__init_tableview__
(
self
):
...
@@ -66,7 +76,9 @@ class AnalyseUim(QObject):
...
@@ -66,7 +76,9 @@ class AnalyseUim(QObject):
table
,
row_position
=
self
.
__add_row_in_table__
()
table
,
row_position
=
self
.
__add_row_in_table__
()
table
.
setItem
(
row_position
,
0
,
QTableWidgetItem
(
message
))
table
.
setItem
(
row_position
,
0
,
QTableWidgetItem
(
message
))
def
__display_step_with_button__
(
self
,
message
:
str
,
button_text
:
str
,
button_action
:
str
):
def
__display_step_with_button__
(
self
,
message
:
str
,
button_text
:
str
,
button_action
:
str
):
table
,
row_position
=
self
.
__add_row_in_table__
()
table
,
row_position
=
self
.
__add_row_in_table__
()
table
.
setItem
(
row_position
,
0
,
QTableWidgetItem
(
message
))
table
.
setItem
(
row_position
,
0
,
QTableWidgetItem
(
message
))
button
=
QPushButton
(
button_text
)
button
=
QPushButton
(
button_text
)
...
@@ -106,6 +118,9 @@ class AnalyseUim(QObject):
...
@@ -106,6 +118,9 @@ class AnalyseUim(QObject):
def
__open_pic_conduc_height_df__
(
self
):
def
__open_pic_conduc_height_df__
(
self
):
os
.
system
(
"
xdg-open
"
+
self
.
ctrl
.
pic_conduc_height_filepath
)
os
.
system
(
"
xdg-open
"
+
self
.
ctrl
.
pic_conduc_height_filepath
)
def
__edit_calib_df__
(
self
):
self
.
__edit_dataframe_in_qdialog__
(
self
.
ctrl
.
calib_df
)
def
__view_calib_df__
(
self
):
def
__view_calib_df__
(
self
):
self
.
__view_dataframe_in_qdialog__
(
self
.
ctrl
.
calib_df
)
self
.
__view_dataframe_in_qdialog__
(
self
.
ctrl
.
calib_df
)
...
@@ -125,9 +140,22 @@ class AnalyseUim(QObject):
...
@@ -125,9 +140,22 @@ class AnalyseUim(QObject):
my_dialog
.
setLayout
(
layout
)
my_dialog
.
setLayout
(
layout
)
my_dialog
.
show
()
my_dialog
.
show
()
def
__edit_dataframe_in_qdialog__
(
self
,
dataframe
:
pd
.
DataFrame
):
self
.
model
=
EditableTableModel
(
dataframe
)
self
.
model
.
dataChanged
.
connect
(
self
.
__update_model_df__
)
self
.
table
=
QTableView
()
self
.
table
.
setModel
(
self
.
model
)
my_dialog
=
QDialog
(
self
.
main_window
)
layout
=
QVBoxLayout
()
layout
.
addWidget
(
self
.
table
)
my_dialog
.
setLayout
(
layout
)
my_dialog
.
show
()
class
TableModel
(
QAbstractTableModel
):
def
__update_model_df__
(
self
):
self
.
ctrl
.
update_calib_csv
(
self
.
model
.
getData
())
class
TableModel
(
QAbstractTableModel
):
def
__init__
(
self
,
data
):
def
__init__
(
self
,
data
):
super
(
TableModel
,
self
).
__init__
()
super
(
TableModel
,
self
).
__init__
()
self
.
_data
=
data
self
.
_data
=
data
...
@@ -150,4 +178,36 @@ class TableModel(QAbstractTableModel):
...
@@ -150,4 +178,36 @@ class TableModel(QAbstractTableModel):
return
str
(
self
.
_data
.
columns
[
section
])
return
str
(
self
.
_data
.
columns
[
section
])
if
orientation
==
Qt
.
Vertical
:
if
orientation
==
Qt
.
Vertical
:
return
str
(
self
.
_data
.
index
[
section
])
return
str
(
self
.
_data
.
index
[
section
])
\ No newline at end of file
class
EditableTableModel
(
TableModel
):
def
__init__
(
self
,
data
):
super
(
EditableTableModel
,
self
).
__init__
(
data
)
def
getData
(
self
):
return
self
.
_data
def
setData
(
self
,
index
,
value
,
role
):
if
not
index
.
isValid
():
return
False
if
role
!=
Qt
.
EditRole
:
return
False
row
=
index
.
row
()
if
row
<
0
or
row
>=
len
(
self
.
_data
.
values
):
return
False
column
=
index
.
column
()
if
column
<
0
or
column
>=
self
.
_data
.
columns
.
size
:
return
False
self
.
_data
.
iloc
[
row
,
column
]
=
value
self
.
dataChanged
.
emit
(
index
,
index
)
return
True
def
flags
(
self
,
index
):
flags
=
super
(
self
.
__class__
,
self
).
flags
(
index
)
flags
|=
Qt
.
ItemIsEditable
flags
|=
Qt
.
ItemIsSelectable
flags
|=
Qt
.
ItemIsEnabled
flags
|=
Qt
.
ItemIsDragEnabled
flags
|=
Qt
.
ItemIsDropEnabled
return
flags
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment