Skip to content

GitLab

Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 1d2efd3a authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production' 

Release 0.6.4

See merge request !29
parents dce38ab3 0f84f4ef
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 2057 deletions
controllers/plugin_event.py
View file @ 1d2efd3a
......@@ -28,25 +28,15 @@ def graph_mpl():
"""Plot list or metric using the matplotlib library
"""
# selector and configuration
try:
ui_table = virtdb.selector
selector = EvtSelector(ui_table)
except SelectorActiveItemsException as e:
return INLINE_ALERT % (T("Error..."), T(str(e)))
config = db.graphs[request.vars.id_graphs]
# build the graph
try:
graph = Graph(config, selector)
graph = Graph(db.graphs, request.vars.id_graphs)
except (IndexError, TypeError, ValueError) as e:
return INLINE_ALERT % ("Error", e)
return INLINE_ALERT % ("GRAPH Error", e)
# build the report title
title = do_title(config, selector)
title = do_title(graph)
# extract the image
# encode special character to be used in the image URI
......@@ -81,39 +71,29 @@ def grid():
The identifier is used to instantiate the proper class.
"""
try:
ui_table = virtdb.selector
selector = EvtSelector(ui_table)
except SelectorActiveItemsException as e:
return INLINE_ALERT % (T("Error..."), T(str(e)))
# extract the report configuration and the build the report
# build the report
try:
if "id_lists" in request.vars:
config = db.lists[request.vars.id_lists]
report = List(config, selector)
report = List(db.lists, request.vars.id_lists)
elif "id_metrics1d" in request.vars:
config = db.metrics1d[request.vars.id_metrics1d]
report = Metric1D(config, selector)
report = Metric1D(db.metrics1d, request.vars.id_metrics1d)
elif "id_metrics2d" in request.vars:
config = db.metrics2d[request.vars.id_metrics2d]
report = Metric2D(config, selector)
report = Metric2D(db.metrics2d, request.vars.id_metrics2d)
else:
return INLINE_ALERT % ("Error", "Report id is not defined.")
except (IndexError, ReportException, TypeError, ValueError) as e:
return INLINE_ALERT % ("JSON Error...", e)
return INLINE_ALERT % ("REPORT Error...", e)
# extract the configurations for the Ext.data.Store and Dbui.grid.Panel.
store = report.to_store()
grid = report.to_grid()
# build the report title
title = do_title(config, selector)
title = do_title(report)
# delegate the grid rendering to the view
return dict(cfg_store=store, grid=grid, title=title)
......
modules/plugin_event/report_objects.py
View file @ 1d2efd3a
......@@ -13,29 +13,34 @@ from pandas import DataFrame, MultiIndex, to_datetime
from plugin_dbui import get_id, Store
from pydal.helpers.methods import smart_query
from pydal.objects import FieldVirtual
from selector import EvtSelector
from StringIO import StringIO
MSG_NO_DATAINDEX = "The property dataIndex is required when eval is used."
MSG_NO_EVT_ID = "Identifier of the event is not defined."
MSG_NO_XTYPE = "The property xtype is missing."
REG_DBFIELD = re.compile("\w+\.\w+(?:\.\w+)?", re.UNICODE)
REG_EVT_ID = re.compile("history\.id_events *={1,2} *(\d+)")
REG_PYQUERY = re.compile("[\( ]*\w+\.\w+\.\w+")
REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE)
def do_title(config, selector):
def do_title(report):
"""Build the report title.
Args:
config (gluon.dal.Row): the list configuration parameter.
selector (EvtSelector): selector handling period of time.
report (BaseReport): the report
Returns:
str:
"""
db = current.globalenv["db"]
db = report.db
config = report.config
selector = report.selector
T = current.T
# from the configuration
......@@ -150,40 +155,55 @@ class BaseReport(object):
"""Base class to build list, metric or graph reports.
Args:
config (gluon.dal.Row): the configuration parameter for the report.
selector (EvtSelector): the selector handling user criteria.
table (gluon.dal.Table): table containing configurations for reports.
id_report (int): identifier of the report in the table.
"""
def __init__(self, config, selector):
def __init__(self, table, id_report):
db = current.globalenv["db"]
db = table._db
self.db = db
self.df = None
self.config = config
self.rows = None
self.selector = selector
# Extract the configuration for the report configuration
config = table[id_report]
# Extract the event identifier located in the condition field
conditions = config.conditions
mtch = REG_EVT_ID.search(conditions)
if mtch is None:
raise ReportException(current.T(MSG_NO_EVT_ID))
id_event = int(mtch.group(1))
# Instantiate the selector
virtdb = current.globalenv["virtdb"]
selector = EvtSelector(virtdb.selector, id_event)
# apply the condition criteria used to filter the history records
# condition can be written as a smart query: history.id_events == 7
# or like a python query: db.events.event == "People"
if "conditions" in config:
condition = config.conditions
# minimal protection to avoid injection flow
# the beginning of the python query should be like:
# db.table.field
# (db.table.field
# ((db.table.field
# ( ( db.table.field
#
if REG_PYQUERY.match(condition):
q_conditions = eval(condition, None, {"db": db})
else:
q_conditions = smart_query(db.history, config.conditions)
# minimal protection to avoid injection flow
# the beginning of the python query should be like:
# db.table.field
# (db.table.field
# ((db.table.field
# ( ( db.table.field
#
if REG_PYQUERY.match(conditions):
q_conditions = eval(conditions, None, {"db": db})
selector.append_query(q_conditions)
else:
q_conditions = smart_query(db.history, conditions)
selector.append_query(q_conditions)
# keep track of configuration and selector
self.config = config
self.selector = selector
def _do_data(self, maps):
"""Build a temporarily list with the raw data for each series.
......@@ -274,19 +294,23 @@ class Graph(BaseReport):
scater plots, *etc*.
Args:
config (gluon.dal.Row):
the configuration parameter for the graph.
table (gluon.dal.Table):
table containing configurations for reports.
selector (EvtSelector):
selector handling period of time.
id_report (int):
identifier of the report in the table.
backend (str):
the name of the matplotlib backend uses to produce figure.
"""
def __init__(self, config, selector, backend="Agg"):
def __init__(self, table, id_report, backend="Agg"):
BaseReport.__init__(self, config, selector)
self.db = table._db
self.config = config = table[id_report]
self.df = None
self.rows = None
# set the matplotlib back end
#
......@@ -315,18 +339,18 @@ class Graph(BaseReport):
report_type = config.report_type
report_name = config.report_name
report_id = get_id(db[report_type], name=report_name)
report_config = db[report_type][report_id]
if report_type == "lists":
report = List(report_config, selector)
report = List(db.lists, report_id)
elif report_type == "metrics1d":
report = Metric1D(report_config, selector)
report = Metric1D(db.metrics1d, report_id)
elif report_type == "metrics2d":
report = Metric2D(report_config, selector)
report = Metric2D(db.metrics2d, report_id)
self.df = report.to_df()
self.selector = report.selector
# build the graph from the DataFrame
self._do_graph()
......@@ -458,16 +482,16 @@ class List(BaseReport):
Its configuration is returned by the method *to_store*.
Args:
config (gluon.dal.Row): the configuration parameter for the list.
selector (EvtSelector): selector handling period of time.
table (gluon.dal.Table): table containing configurations for reports.
id_report (int): identifier of the report in the table.
"""
def __init__(self, config, selector):
def __init__(self, table, id_report):
BaseReport.__init__(self, config, selector)
BaseReport.__init__(self, table, id_report)
# decode column configuration
columns = [Storage(el) for el in json.loads(config.columns)]
columns = [Storage(el) for el in json.loads(self.config.columns)]
# check column configuration
# add database field map (tablename, fieldname, keyname)
......@@ -789,13 +813,15 @@ class Metric1D(List):
A summary information can also be computed for each column or rows.
Args:
config (gluon.dal.Row): the configuration parameter for the metric.
selector (EvtSelector): selector handling period of time.
table (gluon.dal.Table): table containing configurations for reports.
id_report (int): identifier of the report in the table.
"""
def __init__(self, config, selector):
def __init__(self, table, id_report):
BaseReport.__init__(self, config, selector)
BaseReport.__init__(self, table, id_report)
config = self.config
# group by attributes
field_groupby = config.group_field
......@@ -1001,13 +1027,13 @@ class Metric2D(BaseReport):
In fact, all the computation methods of the ``pandas.DataFrame`` class.
Args:
config (gluon.dal.Row): the configuration parameter for the metric.
selector (EvtSelector): selector handling period of time.
table (gluon.dal.Table): table containing configurations for reports.
id_report (int): identifier of the report in the table.
"""
def __init__(self, config, selector):
def __init__(self, table, id_report):
BaseReport.__init__(self, config, selector)
BaseReport.__init__(self, table, id_report)
self._do_metric()
# replace undefined value by 0
......
modules/plugin_event/selector.py
View file @ 1d2efd3a
......@@ -129,7 +129,7 @@ class EvtSelector(SelectorActiveItems):
"""Main selector to build list, metric or graph according to
the user criteria:
- focus on history record
- focus on history record and on a given event.
- Select record active during the selected period of time.
- Selection can be performed on category defined as a string.
- For each record computes virtual field: ``age``, ``coverage``,
......@@ -144,8 +144,10 @@ class EvtSelector(SelectorActiveItems):
categories, year_end, year_start and year are systematically
excluded.
id_event (int): the event identifier
"""
def __init__(self, table, exclude_fields=()):
def __init__(self, table, id_event=0, exclude_fields=()):
li = ["data",
"id_object_categories",
......@@ -158,9 +160,10 @@ class EvtSelector(SelectorActiveItems):
li.extend(exclude_fields)
SelectorActiveItems.__init__(self, table, exclude_fields=li)
# add virtual fields
self._db = db = current.globalenv["db"]
self._id_event = id_event
# add virtual fields
history = db.history
virtual = Field.Virtual
......@@ -172,13 +175,9 @@ class EvtSelector(SelectorActiveItems):
history.is_over = virtual("is_over", self._is_over, ftype="boolean")
history.is_start = virtual("is_start", self._is_start, ftype="boolean")
# keep track of the identifier of the people event
# since it is used by period_activity
self._id_event_people = get_id(db.events, event="People")
def _active_period(self, id_people, id_domain, id_team):
"""Determine the period when a person is active in a given
domain and team.
domain and team and for a given event
Args:
id_people (int):
......@@ -191,7 +190,7 @@ class EvtSelector(SelectorActiveItems):
stop (datetime or None)
"""
rawsql = RAWSQL_ACTIVITY % (self._id_event_people,
rawsql = RAWSQL_ACTIVITY % (self._id_event,
id_people,
id_domain,
id_team)
......
modules/plugin_event/ui_selector.py
View file @ 1d2efd3a
......@@ -54,15 +54,21 @@ class SelectorUi(object):
mdf.configure_field("id_domains", emptyText=text, xtype=mytype)
mdf.configure_field("id_fundings", emptyText=text, xtype=mytype)
# to have an unique key use xcomboboxmaster instead of userreset
mdf.configure_field("id_object_categories",
emptyText=text,
xtype=mytype)
refStore="alias_object_categoriesStore",
userReset=True,
xtype="xcomboboxmaster")
mdf.configure_field("id_object_code", emptyText=text, xtype=mytype)
# to have an unique key use xcomboboxmaster instead of userreset
mdf.configure_field("id_people_categories",
emptyText=text,
xtype=mytype)
refStore="alias_people_categoriesStore",
userReset=True,
xtype="xcomboboxmaster")
mdf.configure_field("id_people_code", emptyText=text, xtype=mytype)
mdf.configure_field("id_teams", emptyText=text, xtype=mytype)
......
scripts/lbfr_people_author_moa.py deleted 100644 → 0
View file @ dce38ab3
# -*- coding: utf-8 -*-
""" NAME
lbfr_people_author_moa
SYNOPSIS
check database consistency between people, author and moa events.
DESCRIPTION
In the track_lhcbfrance application, three events are defined: people
authors and M&O A. The script verify the consistency between them.
OPTIONS
-h, --help
Display the help and exit.
EXAMPLE
> cd ...track_events/scripts
> ./run -S test_lhcbfrance script lbfr_people_author_moa.py
> ./run -S track_lhcbfrance script lbfr_people_author_moa.py
AUTHOR
R. Le Gac -- Jan 2016
"""
from datetime import timedelta
AUTHORS = ['Chercheur',
'Emérite',
'Enseignant-chercheur',
'PhdStudent',
'PostDoc']
MOA = ['Chercheur',
'Emérite',
'Enseignant-chercheur',
'PostDoc']
def check_authors():
"""Check the events consistency for people belonging to the author list.
Author event starts 6 months after the people arrival and ends
one year after its departure.
"""
check_events(get_id(db.projects, project="LHCb"),
get_category_ids(AUTHORS),
"Author",
timedelta(days=182.),
timedelta(days=365))
def check_events(id_project,
people_categorie_ids,
event,
delay_start=timedelta(0.),
delay_end=timedelta(0.)):
"""check the consistency between "people event" and the one selected
in the arguments.
For people having an entry in the "people event" subset, the algorithm
checks that an "event" exits and that its start / end dates agree with
the "people event" ones.
The list of people categories is used to select a subset of
"people events", for example those related to authors, ...
Args:
id_project (int): identifier of the project.
people_categorie_ids (list): list of people categories to build a
subset of "people event".
event (str): the name of the event for which the consistency is checked.
delay_start (timedelta): the delay between the start date of the "people
event" and the "event"
delay_end (timedelta): the delay between the end date of the "people
event" and the "event".
"""
q_cat = db.history.id_people_categories.belongs(people_categorie_ids)
q_project = db.history.id_projects == id_project
id_evt_event = get_id(db.events, event=event)
q_evt_event = (db.history.id_events == id_evt_event) & (q_project)
id_evt_people = get_id(db.events, event="People")
q_evt_people = (db.history.id_events == id_evt_people)
q_evt_people &= (q_cat) & (q_project)
# scan the list of people
for person in db(db.people).select():
id_people = person.id
if id_people == UNDEF_ID:
continue
last_name = db.people[id_people].last_name
# look for "people event(s)" associates to the person
# determine start and end dates of people event
tpl = evt_people(q_evt_people, id_people)
if len(tpl) == 0:
continue
start_date, end_date, team_ids = tpl
# computed the expected dates for the event "event"
exp_start = (None if start_date is None else start_date + delay_start)
exp_end = (None if end_date is None else end_date + delay_end)
# check the event "event"
query = (q_evt_event) & (db.history.id_people == id_people)
myset = db(query)
nrows = myset.count()
# signal too many event
if nrows > 1:
print last_name, start_date, end_date, "→ to many %s event!\n" % event
print "Have a look and fix it by hand.\n"
continue
# the event is missing, add it
if nrows == 0:
print last_name, start_date, end_date, '→ no %s event' % event
if len(team_ids) != 1:
print "Team change during the period."
print "Have a look and fix it by hand.\n"
continue
data = dict(id_events=id_evt_event,
id_people=id_people,
id_projects=id_project,
id_teams=list(team_ids)[0],
start_date= exp_start,
end_date=exp_end)
fix_event(**data)
continue
# the event exits
row = myset.select(db.history.ALL).first()
evt_start, evt_end = row.start_date, row.end_date
# check the start date
# deal with special case
ok_start = evt_start is None and start_date is None
ok_start |= evt_start <= start_date
ok_start |= evt_start == exp_start
if not ok_start:
print last_name, "→ wrong %s start date" % event
print "The person start :", start_date
print "The event start :", evt_start
print "The expected start:", exp_start
if evt_start and exp_start:
print "Delta [days] :", (evt_start - exp_start).days
data = dict(id=row.id, start_date= exp_start)
fix_event(**data)
# check the end date
ok_end = evt_end is None and end_date is None
ok_end |= evt_end == exp_end
if not ok_end:
print last_name, "→ wrong %s end date" % event
print "The person quit :", end_date
print "The event end :", evt_end
print "The expected end:", exp_end
if evt_end and exp_end:
print "Delta [days] :", (evt_end - exp_end).days
data = dict(id=row.id, end_date= exp_end)
fix_event(**data)
def check_moa():
"""Check the events consistency for people belonging to the M&O A list.
"""
check_events(get_id(db.projects, project="LHCb"),
get_category_ids(MOA),
"M&O Cat A",
timedelta(days=0),
timedelta(days=0))
def evt_people(query_event_people, id_people):
"""For a given person determine the minimum start date and the
maximum end date of the people event. It also extract the list
of team ids.
Args:
query_event_people(gluon.dal.Query): the query to select all the
"people event" or a subset of them.
id_people (int): the identifier of the person in the people table.
Returns:
tuple: containing the start and the end dates. It is empty
when there is no people events associated to the person.
The last element is the list of id_teams.
"""
start_dates, end_dates, team_ids = set(), set(), set()
query = (query_event_people) & (db.history.id_people == id_people)
myset = db(query)
if myset.count() == 0:
return ()
for row in myset.select(db.history.ALL):
start_dates.add(row.start_date)
end_dates.add(row.end_date)
team_ids.add(row.id_teams)
start_date = (None if None in start_dates else min(start_dates))
end_date = (None if None in end_dates else max(end_dates))
return (start_date, end_date, team_ids)
def fix_event(**kwargs):
"""Fix the event.
Insert of update the event.
Keyword arguments: