Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
tev
plugin_event
Commits
1d2efd3a
Commit
1d2efd3a
authored
Jan 13, 2017
by
LE GAC Renaud
Browse files
Merge branch 'master' into 'production'
Release 0.6.4 See merge request
!29
parents
dce38ab3
0f84f4ef
Changes
9
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
108 additions
and
2057 deletions
+108
-2057
controllers/plugin_event.py
controllers/plugin_event.py
+9
-29
modules/plugin_event/report_objects.py
modules/plugin_event/report_objects.py
+74
-48
modules/plugin_event/selector.py
modules/plugin_event/selector.py
+8
-9
modules/plugin_event/ui_selector.py
modules/plugin_event/ui_selector.py
+8
-2
scripts/lbfr_people_author_moa.py
scripts/lbfr_people_author_moa.py
+0
-307
scripts/lbfr_report.py
scripts/lbfr_report.py
+0
-1659
static/plugin_event/CHANGELOG
static/plugin_event/CHANGELOG
+6
-0
static/plugin_event/VERSION
static/plugin_event/VERSION
+1
-1
views/plugin_event/grid.html
views/plugin_event/grid.html
+2
-2
No files found.
controllers/plugin_event.py
View file @
1d2efd3a
...
...
@@ -28,25 +28,15 @@ def graph_mpl():
"""Plot list or metric using the matplotlib library
"""
# selector and configuration
try
:
ui_table
=
virtdb
.
selector
selector
=
EvtSelector
(
ui_table
)
except
SelectorActiveItemsException
as
e
:
return
INLINE_ALERT
%
(
T
(
"Error..."
),
T
(
str
(
e
)))
config
=
db
.
graphs
[
request
.
vars
.
id_graphs
]
# build the graph
try
:
graph
=
Graph
(
config
,
selector
)
graph
=
Graph
(
db
.
graphs
,
request
.
vars
.
id_graphs
)
except
(
IndexError
,
TypeError
,
ValueError
)
as
e
:
return
INLINE_ALERT
%
(
"Error"
,
e
)
return
INLINE_ALERT
%
(
"
GRAPH
Error"
,
e
)
# build the report title
title
=
do_title
(
config
,
selector
)
title
=
do_title
(
graph
)
# extract the image
# encode special character to be used in the image URI
...
...
@@ -81,39 +71,29 @@ def grid():
The identifier is used to instantiate the proper class.
"""
try
:
ui_table
=
virtdb
.
selector
selector
=
EvtSelector
(
ui_table
)
except
SelectorActiveItemsException
as
e
:
return
INLINE_ALERT
%
(
T
(
"Error..."
),
T
(
str
(
e
)))
# extract the report configuration and the build the report
# build the report
try
:
if
"id_lists"
in
request
.
vars
:
config
=
db
.
lists
[
request
.
vars
.
id_lists
]
report
=
List
(
config
,
selector
)
report
=
List
(
db
.
lists
,
request
.
vars
.
id_lists
)
elif
"id_metrics1d"
in
request
.
vars
:
config
=
db
.
metrics1d
[
request
.
vars
.
id_metrics1d
]
report
=
Metric1D
(
config
,
selector
)
report
=
Metric1D
(
db
.
metrics1d
,
request
.
vars
.
id_metrics1d
)
elif
"id_metrics2d"
in
request
.
vars
:
config
=
db
.
metrics2d
[
request
.
vars
.
id_metrics2d
]
report
=
Metric2D
(
config
,
selector
)
report
=
Metric2D
(
db
.
metrics2d
,
request
.
vars
.
id_metrics2d
)
else
:
return
INLINE_ALERT
%
(
"Error"
,
"Report id is not defined."
)
except
(
IndexError
,
ReportException
,
TypeError
,
ValueError
)
as
e
:
return
INLINE_ALERT
%
(
"
JSON
Error..."
,
e
)
return
INLINE_ALERT
%
(
"
REPORT
Error..."
,
e
)
# extract the configurations for the Ext.data.Store and Dbui.grid.Panel.
store
=
report
.
to_store
()
grid
=
report
.
to_grid
()
# build the report title
title
=
do_title
(
config
,
select
or
)
title
=
do_title
(
rep
or
t
)
# delegate the grid rendering to the view
return
dict
(
cfg_store
=
store
,
grid
=
grid
,
title
=
title
)
...
...
modules/plugin_event/report_objects.py
View file @
1d2efd3a
...
...
@@ -13,29 +13,34 @@ from pandas import DataFrame, MultiIndex, to_datetime
from
plugin_dbui
import
get_id
,
Store
from
pydal.helpers.methods
import
smart_query
from
pydal.objects
import
FieldVirtual
from
selector
import
EvtSelector
from
StringIO
import
StringIO
MSG_NO_DATAINDEX
=
"The property dataIndex is required when eval is used."
MSG_NO_EVT_ID
=
"Identifier of the event is not defined."
MSG_NO_XTYPE
=
"The property xtype is missing."
REG_DBFIELD
=
re
.
compile
(
"\w+\.\w+(?:\.\w+)?"
,
re
.
UNICODE
)
REG_EVT_ID
=
re
.
compile
(
"history\.id_events *={1,2} *(\d+)"
)
REG_PYQUERY
=
re
.
compile
(
"[\( ]*\w+\.\w+\.\w+"
)
REG_SINGLE_DBFIELD
=
re
.
compile
(
"^ *\w+\.\w+(\.\w+)? *$"
,
re
.
UNICODE
)
def
do_title
(
config
,
select
or
):
def
do_title
(
rep
or
t
):
"""Build the report title.
Args:
config (gluon.dal.Row): the list configuration parameter.
selector (EvtSelector): selector handling period of time.
report (BaseReport): the report
Returns:
str:
"""
db
=
current
.
globalenv
[
"db"
]
db
=
report
.
db
config
=
report
.
config
selector
=
report
.
selector
T
=
current
.
T
# from the configuration
...
...
@@ -150,40 +155,55 @@ class BaseReport(object):
"""Base class to build list, metric or graph reports.
Args:
config
(gluon.dal.
Row
): t
h
e con
figuration parameter
for
the
report.
select
or (
EvtSelector): the selector handling user criteria
.
table
(gluon.dal.
Table
): t
abl
e con
taining configurations
for report
s
.
id_rep
or
t
(
int): identifier of the report in the table
.
"""
def
__init__
(
self
,
config
,
select
or
):
def
__init__
(
self
,
table
,
id_rep
or
t
):
db
=
current
.
globalenv
[
"db"
]
db
=
table
.
_db
self
.
db
=
db
self
.
df
=
None
self
.
config
=
config
self
.
rows
=
None
self
.
selector
=
selector
# Extract the configuration for the report configuration
config
=
table
[
id_report
]
# Extract the event identifier located in the condition field
conditions
=
config
.
conditions
mtch
=
REG_EVT_ID
.
search
(
conditions
)
if
mtch
is
None
:
raise
ReportException
(
current
.
T
(
MSG_NO_EVT_ID
))
id_event
=
int
(
mtch
.
group
(
1
))
# Instantiate the selector
virtdb
=
current
.
globalenv
[
"virtdb"
]
selector
=
EvtSelector
(
virtdb
.
selector
,
id_event
)
# apply the condition criteria used to filter the history records
# condition can be written as a smart query: history.id_events == 7
# or like a python query: db.events.event == "People"
if
"conditions"
in
config
:
condition
=
config
.
conditions
# minimal protection to avoid injection flow
# the beginning of the python query should be like:
# db.table.field
# (db.table.field
# ((db.table.field
# ( ( db.table.field
#
if
REG_PYQUERY
.
match
(
condition
):
q_conditions
=
eval
(
condition
,
None
,
{
"db"
:
db
})
else
:
q_conditions
=
smart_query
(
db
.
history
,
config
.
conditions
)
# minimal protection to avoid injection flow
# the beginning of the python query should be like:
# db.table.field
# (db.table.field
# ((db.table.field
# ( ( db.table.field
#
if
REG_PYQUERY
.
match
(
conditions
):
q_conditions
=
eval
(
conditions
,
None
,
{
"db"
:
db
})
selector
.
append_query
(
q_conditions
)
else
:
q_conditions
=
smart_query
(
db
.
history
,
conditions
)
selector
.
append_query
(
q_conditions
)
# keep track of configuration and selector
self
.
config
=
config
self
.
selector
=
selector
def
_do_data
(
self
,
maps
):
"""Build a temporarily list with the raw data for each series.
...
...
@@ -274,19 +294,23 @@ class Graph(BaseReport):
scater plots, *etc*.
Args:
config
(gluon.dal.
Row
):
t
h
e con
figuration parameter for the graph
.
table
(gluon.dal.
Table
):
t
abl
e con
taining configurations for reports
.
select
or (
EvtSelector
):
selector handling period of tim
e.
id_rep
or
t
(
int
):
identifier of the report in the tabl
e.
backend (str):
the name of the matplotlib backend uses to produce figure.
"""
def
__init__
(
self
,
config
,
select
or
,
backend
=
"Agg"
):
def
__init__
(
self
,
table
,
id_rep
or
t
,
backend
=
"Agg"
):
BaseReport
.
__init__
(
self
,
config
,
selector
)
self
.
db
=
table
.
_db
self
.
config
=
config
=
table
[
id_report
]
self
.
df
=
None
self
.
rows
=
None
# set the matplotlib back end
#
...
...
@@ -315,18 +339,18 @@ class Graph(BaseReport):
report_type
=
config
.
report_type
report_name
=
config
.
report_name
report_id
=
get_id
(
db
[
report_type
],
name
=
report_name
)
report_config
=
db
[
report_type
][
report_id
]
if
report_type
==
"lists"
:
report
=
List
(
report_config
,
selector
)
report
=
List
(
db
.
lists
,
report_id
)
elif
report_type
==
"metrics1d"
:
report
=
Metric1D
(
report_config
,
selector
)
report
=
Metric1D
(
db
.
metrics1d
,
report_id
)
elif
report_type
==
"metrics2d"
:
report
=
Metric2D
(
report_config
,
selector
)
report
=
Metric2D
(
db
.
metrics2d
,
report_id
)
self
.
df
=
report
.
to_df
()
self
.
selector
=
report
.
selector
# build the graph from the DataFrame
self
.
_do_graph
()
...
...
@@ -458,16 +482,16 @@ class List(BaseReport):
Its configuration is returned by the method *to_store*.
Args:
config
(gluon.dal.
Row
): t
h
e con
figuration parameter for the list
.
select
or (
EvtSelector): selector handling period of tim
e.
table
(gluon.dal.
Table
): t
abl
e con
taining configurations for reports
.
id_rep
or
t
(
int): identifier of the report in the tabl
e.
"""
def
__init__
(
self
,
config
,
select
or
):
def
__init__
(
self
,
table
,
id_rep
or
t
):
BaseReport
.
__init__
(
self
,
config
,
select
or
)
BaseReport
.
__init__
(
self
,
table
,
id_rep
or
t
)
# decode column configuration
columns
=
[
Storage
(
el
)
for
el
in
json
.
loads
(
config
.
columns
)]
columns
=
[
Storage
(
el
)
for
el
in
json
.
loads
(
self
.
config
.
columns
)]
# check column configuration
# add database field map (tablename, fieldname, keyname)
...
...
@@ -789,13 +813,15 @@ class Metric1D(List):
A summary information can also be computed for each column or rows.
Args:
config
(gluon.dal.
Row
): t
h
e con
figuration parameter for the metric
.
select
or (
EvtSelector): selector handling period of tim
e.
table
(gluon.dal.
Table
): t
abl
e con
taining configurations for reports
.
id_rep
or
t
(
int): identifier of the report in the tabl
e.
"""
def
__init__
(
self
,
config
,
select
or
):
def
__init__
(
self
,
table
,
id_rep
or
t
):
BaseReport
.
__init__
(
self
,
config
,
selector
)
BaseReport
.
__init__
(
self
,
table
,
id_report
)
config
=
self
.
config
# group by attributes
field_groupby
=
config
.
group_field
...
...
@@ -1001,13 +1027,13 @@ class Metric2D(BaseReport):
In fact, all the computation methods of the ``pandas.DataFrame`` class.
Args:
config
(gluon.dal.
Row
): t
h
e con
figuration parameter for the metric
.
select
or (
EvtSelector): selector handling period of tim
e.
table
(gluon.dal.
Table
): t
abl
e con
taining configurations for reports
.
id_rep
or
t
(
int): identifier of the report in the tabl
e.
"""
def
__init__
(
self
,
config
,
select
or
):
def
__init__
(
self
,
table
,
id_rep
or
t
):
BaseReport
.
__init__
(
self
,
config
,
select
or
)
BaseReport
.
__init__
(
self
,
table
,
id_rep
or
t
)
self
.
_do_metric
()
# replace undefined value by 0
...
...
modules/plugin_event/selector.py
View file @
1d2efd3a
...
...
@@ -129,7 +129,7 @@ class EvtSelector(SelectorActiveItems):
"""Main selector to build list, metric or graph according to
the user criteria:
- focus on history record
- focus on history record
and on a given event.
- Select record active during the selected period of time.
- Selection can be performed on category defined as a string.
- For each record computes virtual field: ``age``, ``coverage``,
...
...
@@ -144,8 +144,10 @@ class EvtSelector(SelectorActiveItems):
categories, year_end, year_start and year are systematically
excluded.
id_event (int): the event identifier
"""
def
__init__
(
self
,
table
,
exclude_fields
=
()):
def
__init__
(
self
,
table
,
id_event
=
0
,
exclude_fields
=
()):
li
=
[
"data"
,
"id_object_categories"
,
...
...
@@ -158,9 +160,10 @@ class EvtSelector(SelectorActiveItems):
li
.
extend
(
exclude_fields
)
SelectorActiveItems
.
__init__
(
self
,
table
,
exclude_fields
=
li
)
# add virtual fields
self
.
_db
=
db
=
current
.
globalenv
[
"db"
]
self
.
_id_event
=
id_event
# add virtual fields
history
=
db
.
history
virtual
=
Field
.
Virtual
...
...
@@ -172,13 +175,9 @@ class EvtSelector(SelectorActiveItems):
history
.
is_over
=
virtual
(
"is_over"
,
self
.
_is_over
,
ftype
=
"boolean"
)
history
.
is_start
=
virtual
(
"is_start"
,
self
.
_is_start
,
ftype
=
"boolean"
)
# keep track of the identifier of the people event
# since it is used by period_activity
self
.
_id_event_people
=
get_id
(
db
.
events
,
event
=
"People"
)
def
_active_period
(
self
,
id_people
,
id_domain
,
id_team
):
"""Determine the period when a person is active in a given
domain and team
.
domain and team
and for a given event
Args:
id_people (int):
...
...
@@ -191,7 +190,7 @@ class EvtSelector(SelectorActiveItems):
stop (datetime or None)
"""
rawsql
=
RAWSQL_ACTIVITY
%
(
self
.
_id_event
_people
,
rawsql
=
RAWSQL_ACTIVITY
%
(
self
.
_id_event
,
id_people
,
id_domain
,
id_team
)
...
...
modules/plugin_event/ui_selector.py
View file @
1d2efd3a
...
...
@@ -54,15 +54,21 @@ class SelectorUi(object):
mdf
.
configure_field
(
"id_domains"
,
emptyText
=
text
,
xtype
=
mytype
)
mdf
.
configure_field
(
"id_fundings"
,
emptyText
=
text
,
xtype
=
mytype
)
# to have an unique key use xcomboboxmaster instead of userreset
mdf
.
configure_field
(
"id_object_categories"
,
emptyText
=
text
,
xtype
=
mytype
)
refStore
=
"alias_object_categoriesStore"
,
userReset
=
True
,
xtype
=
"xcomboboxmaster"
)
mdf
.
configure_field
(
"id_object_code"
,
emptyText
=
text
,
xtype
=
mytype
)
# to have an unique key use xcomboboxmaster instead of userreset
mdf
.
configure_field
(
"id_people_categories"
,
emptyText
=
text
,
xtype
=
mytype
)
refStore
=
"alias_people_categoriesStore"
,
userReset
=
True
,
xtype
=
"xcomboboxmaster"
)
mdf
.
configure_field
(
"id_people_code"
,
emptyText
=
text
,
xtype
=
mytype
)
mdf
.
configure_field
(
"id_teams"
,
emptyText
=
text
,
xtype
=
mytype
)
...
...
scripts/lbfr_people_author_moa.py
deleted
100644 → 0
View file @
dce38ab3
# -*- coding: utf-8 -*-
""" NAME
lbfr_people_author_moa
SYNOPSIS
check database consistency between people, author and moa events.
DESCRIPTION
In the track_lhcbfrance application, three events are defined: people
authors and M&O A. The script verify the consistency between them.
OPTIONS
-h, --help
Display the help and exit.
EXAMPLE
> cd ...track_events/scripts
> ./run -S test_lhcbfrance script lbfr_people_author_moa.py
> ./run -S track_lhcbfrance script lbfr_people_author_moa.py
AUTHOR
R. Le Gac -- Jan 2016
"""
from
datetime
import
timedelta
AUTHORS
=
[
'Chercheur'
,
'Emérite'
,
'Enseignant-chercheur'
,
'PhdStudent'
,
'PostDoc'
]
MOA
=
[
'Chercheur'
,
'Emérite'
,
'Enseignant-chercheur'
,
'PostDoc'
]
def
check_authors
():
"""Check the events consistency for people belonging to the author list.
Author event starts 6 months after the people arrival and ends
one year after its departure.
"""
check_events
(
get_id
(
db
.
projects
,
project
=
"LHCb"
),
get_category_ids
(
AUTHORS
),
"Author"
,
timedelta
(
days
=
182.
),
timedelta
(
days
=
365
))
def
check_events
(
id_project
,
people_categorie_ids
,
event
,
delay_start
=
timedelta
(
0.
),
delay_end
=
timedelta
(
0.
)):
"""check the consistency between "people event" and the one selected
in the arguments.
For people having an entry in the "people event" subset, the algorithm
checks that an "event" exits and that its start / end dates agree with
the "people event" ones.
The list of people categories is used to select a subset of
"people events", for example those related to authors, ...
Args:
id_project (int): identifier of the project.
people_categorie_ids (list): list of people categories to build a
subset of "people event".
event (str): the name of the event for which the consistency is checked.
delay_start (timedelta): the delay between the start date of the "people
event" and the "event"
delay_end (timedelta): the delay between the end date of the "people
event" and the "event".
"""
q_cat
=
db
.
history
.
id_people_categories
.
belongs
(
people_categorie_ids
)
q_project
=
db
.
history
.
id_projects
==
id_project
id_evt_event
=
get_id
(
db
.
events
,
event
=
event
)
q_evt_event
=
(
db
.
history
.
id_events
==
id_evt_event
)
&
(
q_project
)
id_evt_people
=
get_id
(
db
.
events
,
event
=
"People"
)
q_evt_people
=
(
db
.
history
.
id_events
==
id_evt_people
)
q_evt_people
&=
(
q_cat
)
&
(
q_project
)
# scan the list of people
for
person
in
db
(
db
.
people
).
select
():
id_people
=
person
.
id
if
id_people
==
UNDEF_ID
:
continue
last_name
=
db
.
people
[
id_people
].
last_name
# look for "people event(s)" associates to the person
# determine start and end dates of people event
tpl
=
evt_people
(
q_evt_people
,
id_people
)
if
len
(
tpl
)
==
0
:
continue
start_date
,
end_date
,
team_ids
=
tpl
# computed the expected dates for the event "event"
exp_start
=
(
None
if
start_date
is
None
else
start_date
+
delay_start
)
exp_end
=
(
None
if
end_date
is
None
else
end_date
+
delay_end
)
# check the event "event"
query
=
(
q_evt_event
)
&
(
db
.
history
.
id_people
==
id_people
)
myset
=
db
(
query
)
nrows
=
myset
.
count
()
# signal too many event
if
nrows
>
1
:
print
last_name
,
start_date
,
end_date
,
"→ to many %s event!
\n
"
%
event
print
"Have a look and fix it by hand.
\n
"
continue
# the event is missing, add it
if
nrows
==
0
:
print
last_name
,
start_date
,
end_date
,
'→ no %s event'
%
event
if
len
(
team_ids
)
!=
1
:
print
"Team change during the period."
print
"Have a look and fix it by hand.
\n
"
continue
data
=
dict
(
id_events
=
id_evt_event
,
id_people
=
id_people
,
id_projects
=
id_project
,
id_teams
=
list
(
team_ids
)[
0
],
start_date
=
exp_start
,
end_date
=
exp_end
)
fix_event
(
**
data
)
continue
# the event exits
row
=
myset
.
select
(
db
.
history
.
ALL
).
first
()
evt_start
,
evt_end
=
row
.
start_date
,
row
.
end_date
# check the start date
# deal with special case
ok_start
=
evt_start
is
None
and
start_date
is
None
ok_start
|=
evt_start
<=
start_date
ok_start
|=
evt_start
==
exp_start
if
not
ok_start
:
print
last_name
,
"→ wrong %s start date"
%
event
print
"The person start :"
,
start_date
print
"The event start :"
,
evt_start
print
"The expected start:"
,
exp_start
if
evt_start
and
exp_start
:
print
"Delta [days] :"
,
(
evt_start
-
exp_start
).
days
data
=
dict
(
id
=
row
.
id
,
start_date
=
exp_start
)
fix_event
(
**
data
)
# check the end date
ok_end
=
evt_end
is
None
and
end_date
is
None
ok_end
|=
evt_end
==
exp_end
if
not
ok_end
:
print
last_name
,
"→ wrong %s end date"
%
event
print
"The person quit :"
,
end_date
print
"The event end :"
,
evt_end
print
"The expected end:"
,
exp_end
if
evt_end
and
exp_end
:
print
"Delta [days] :"
,
(
evt_end
-
exp_end
).
days
data
=
dict
(
id
=
row
.
id
,
end_date
=
exp_end
)
fix_event
(
**
data
)
def
check_moa
():
"""Check the events consistency for people belonging to the M&O A list.
"""
check_events
(
get_id
(
db
.
projects
,
project
=
"LHCb"
),
get_category_ids
(
MOA
),
"M&O Cat A"
,
timedelta
(
days
=
0
),
timedelta
(
days
=
0
))
def
evt_people
(
query_event_people
,
id_people
):
"""For a given person determine the minimum start date and the
maximum end date of the people event. It also extract the list
of team ids.
Args:
query_event_people(gluon.dal.Query): the query to select all the
"people event" or a subset of them.
id_people (int): the identifier of the person in the people table.
Returns:
tuple: containing the start and the end dates. It is empty
when there is no people events associated to the person.
The last element is the list of id_teams.
"""
start_dates
,
end_dates
,
team_ids
=
set
(),
set
(),
set
()
query
=
(
query_event_people
)
&
(
db
.
history
.
id_people
==
id_people
)
myset
=
db
(
query
)
if
myset
.
count
()
==
0
:
return
()
for
row
in
myset
.
select
(
db
.
history
.
ALL
):
start_dates
.
add
(
row
.
start_date
)
end_dates
.
add
(
row
.
end_date
)
team_ids
.
add
(
row
.
id_teams
)
start_date
=
(
None
if
None
in
start_dates
else
min
(
start_dates
))
end_date
=
(
None
if
None
in
end_dates
else
max
(
end_dates
))
return
(
start_date
,
end_date
,
team_ids
)
def
fix_event
(
**
kwargs
):
"""Fix the event.
Insert of update the event.
Keyword arguments:
id (int): id of the event