Commit 1d2efd3a authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production'

Release 0.6.4

See merge request !29
parents dce38ab3 0f84f4ef
...@@ -28,25 +28,15 @@ def graph_mpl(): ...@@ -28,25 +28,15 @@ def graph_mpl():
"""Plot list or metric using the matplotlib library """Plot list or metric using the matplotlib library
""" """
# selector and configuration
try:
ui_table = virtdb.selector
selector = EvtSelector(ui_table)
except SelectorActiveItemsException as e:
return INLINE_ALERT % (T("Error..."), T(str(e)))
config = db.graphs[request.vars.id_graphs]
# build the graph # build the graph
try: try:
graph = Graph(config, selector) graph = Graph(db.graphs, request.vars.id_graphs)
except (IndexError, TypeError, ValueError) as e: except (IndexError, TypeError, ValueError) as e:
return INLINE_ALERT % ("Error", e) return INLINE_ALERT % ("GRAPH Error", e)
# build the report title # build the report title
title = do_title(config, selector) title = do_title(graph)
# extract the image # extract the image
# encode special character to be used in the image URI # encode special character to be used in the image URI
...@@ -81,39 +71,29 @@ def grid(): ...@@ -81,39 +71,29 @@ def grid():
The identifier is used to instantiate the proper class. The identifier is used to instantiate the proper class.
""" """
try: # build the report
ui_table = virtdb.selector
selector = EvtSelector(ui_table)
except SelectorActiveItemsException as e:
return INLINE_ALERT % (T("Error..."), T(str(e)))
# extract the report configuration and the build the report
try: try:
if "id_lists" in request.vars: if "id_lists" in request.vars:
config = db.lists[request.vars.id_lists] report = List(db.lists, request.vars.id_lists)
report = List(config, selector)
elif "id_metrics1d" in request.vars: elif "id_metrics1d" in request.vars:
config = db.metrics1d[request.vars.id_metrics1d] report = Metric1D(db.metrics1d, request.vars.id_metrics1d)
report = Metric1D(config, selector)
elif "id_metrics2d" in request.vars: elif "id_metrics2d" in request.vars:
config = db.metrics2d[request.vars.id_metrics2d] report = Metric2D(db.metrics2d, request.vars.id_metrics2d)
report = Metric2D(config, selector)
else: else:
return INLINE_ALERT % ("Error", "Report id is not defined.") return INLINE_ALERT % ("Error", "Report id is not defined.")
except (IndexError, ReportException, TypeError, ValueError) as e: except (IndexError, ReportException, TypeError, ValueError) as e:
return INLINE_ALERT % ("JSON Error...", e) return INLINE_ALERT % ("REPORT Error...", e)
# extract the configurations for the Ext.data.Store and Dbui.grid.Panel. # extract the configurations for the Ext.data.Store and Dbui.grid.Panel.
store = report.to_store() store = report.to_store()
grid = report.to_grid() grid = report.to_grid()
# build the report title # build the report title
title = do_title(config, selector) title = do_title(report)
# delegate the grid rendering to the view # delegate the grid rendering to the view
return dict(cfg_store=store, grid=grid, title=title) return dict(cfg_store=store, grid=grid, title=title)
......
...@@ -13,29 +13,34 @@ from pandas import DataFrame, MultiIndex, to_datetime ...@@ -13,29 +13,34 @@ from pandas import DataFrame, MultiIndex, to_datetime
from plugin_dbui import get_id, Store from plugin_dbui import get_id, Store
from pydal.helpers.methods import smart_query from pydal.helpers.methods import smart_query
from pydal.objects import FieldVirtual from pydal.objects import FieldVirtual
from selector import EvtSelector
from StringIO import StringIO from StringIO import StringIO
MSG_NO_DATAINDEX = "The property dataIndex is required when eval is used." MSG_NO_DATAINDEX = "The property dataIndex is required when eval is used."
MSG_NO_EVT_ID = "Identifier of the event is not defined."
MSG_NO_XTYPE = "The property xtype is missing." MSG_NO_XTYPE = "The property xtype is missing."
REG_DBFIELD = re.compile("\w+\.\w+(?:\.\w+)?", re.UNICODE) REG_DBFIELD = re.compile("\w+\.\w+(?:\.\w+)?", re.UNICODE)
REG_EVT_ID = re.compile("history\.id_events *={1,2} *(\d+)")
REG_PYQUERY = re.compile("[\( ]*\w+\.\w+\.\w+") REG_PYQUERY = re.compile("[\( ]*\w+\.\w+\.\w+")
REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE) REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE)
def do_title(config, selector): def do_title(report):
"""Build the report title. """Build the report title.
Args: Args:
config (gluon.dal.Row): the list configuration parameter. report (BaseReport): the report
selector (EvtSelector): selector handling period of time.
Returns: Returns:
str: str:
""" """
db = current.globalenv["db"] db = report.db
config = report.config
selector = report.selector
T = current.T T = current.T
# from the configuration # from the configuration
...@@ -150,25 +155,36 @@ class BaseReport(object): ...@@ -150,25 +155,36 @@ class BaseReport(object):
"""Base class to build list, metric or graph reports. """Base class to build list, metric or graph reports.
Args: Args:
config (gluon.dal.Row): the configuration parameter for the report. table (gluon.dal.Table): table containing configurations for reports.
selector (EvtSelector): the selector handling user criteria. id_report (int): identifier of the report in the table.
""" """
def __init__(self, config, selector): def __init__(self, table, id_report):
db = current.globalenv["db"] db = table._db
self.db = db self.db = db
self.df = None self.df = None
self.config = config
self.rows = None self.rows = None
self.selector = selector
# Extract the configuration for the report configuration
config = table[id_report]
# Extract the event identifier located in the condition field
conditions = config.conditions
mtch = REG_EVT_ID.search(conditions)
if mtch is None:
raise ReportException(current.T(MSG_NO_EVT_ID))
id_event = int(mtch.group(1))
# Instantiate the selector
virtdb = current.globalenv["virtdb"]
selector = EvtSelector(virtdb.selector, id_event)
# apply the condition criteria used to filter the history records # apply the condition criteria used to filter the history records
# condition can be written as a smart query: history.id_events == 7 # condition can be written as a smart query: history.id_events == 7
# or like a python query: db.events.event == "People" # or like a python query: db.events.event == "People"
if "conditions" in config:
condition = config.conditions
# minimal protection to avoid injection flow # minimal protection to avoid injection flow
# the beginning of the python query should be like: # the beginning of the python query should be like:
...@@ -177,14 +193,18 @@ class BaseReport(object): ...@@ -177,14 +193,18 @@ class BaseReport(object):
# ((db.table.field # ((db.table.field
# ( ( db.table.field # ( ( db.table.field
# #
if REG_PYQUERY.match(condition): if REG_PYQUERY.match(conditions):
q_conditions = eval(condition, None, {"db": db}) q_conditions = eval(conditions, None, {"db": db})
else: else:
q_conditions = smart_query(db.history, config.conditions) q_conditions = smart_query(db.history, conditions)
selector.append_query(q_conditions) selector.append_query(q_conditions)
# keep track of configuration and selector
self.config = config
self.selector = selector
def _do_data(self, maps): def _do_data(self, maps):
"""Build a temporarily list with the raw data for each series. """Build a temporarily list with the raw data for each series.
This method handle the "year" database field. This method handle the "year" database field.
...@@ -274,19 +294,23 @@ class Graph(BaseReport): ...@@ -274,19 +294,23 @@ class Graph(BaseReport):
scater plots, *etc*. scater plots, *etc*.
Args: Args:
config (gluon.dal.Row): table (gluon.dal.Table):
the configuration parameter for the graph. table containing configurations for reports.
selector (EvtSelector): id_report (int):
selector handling period of time. identifier of the report in the table.
backend (str): backend (str):
the name of the matplotlib backend uses to produce figure. the name of the matplotlib backend uses to produce figure.
""" """
def __init__(self, config, selector, backend="Agg"): def __init__(self, table, id_report, backend="Agg"):
BaseReport.__init__(self, config, selector) self.db = table._db
self.config = config = table[id_report]
self.df = None
self.rows = None
# set the matplotlib back end # set the matplotlib back end
# #
...@@ -315,18 +339,18 @@ class Graph(BaseReport): ...@@ -315,18 +339,18 @@ class Graph(BaseReport):
report_type = config.report_type report_type = config.report_type
report_name = config.report_name report_name = config.report_name
report_id = get_id(db[report_type], name=report_name) report_id = get_id(db[report_type], name=report_name)
report_config = db[report_type][report_id]
if report_type == "lists": if report_type == "lists":
report = List(report_config, selector) report = List(db.lists, report_id)
elif report_type == "metrics1d": elif report_type == "metrics1d":
report = Metric1D(report_config, selector) report = Metric1D(db.metrics1d, report_id)
elif report_type == "metrics2d": elif report_type == "metrics2d":
report = Metric2D(report_config, selector) report = Metric2D(db.metrics2d, report_id)
self.df = report.to_df() self.df = report.to_df()
self.selector = report.selector
# build the graph from the DataFrame # build the graph from the DataFrame
self._do_graph() self._do_graph()
...@@ -458,16 +482,16 @@ class List(BaseReport): ...@@ -458,16 +482,16 @@ class List(BaseReport):
Its configuration is returned by the method *to_store*. Its configuration is returned by the method *to_store*.
Args: Args:
config (gluon.dal.Row): the configuration parameter for the list. table (gluon.dal.Table): table containing configurations for reports.
selector (EvtSelector): selector handling period of time. id_report (int): identifier of the report in the table.
""" """
def __init__(self, config, selector): def __init__(self, table, id_report):
BaseReport.__init__(self, config, selector) BaseReport.__init__(self, table, id_report)
# decode column configuration # decode column configuration
columns = [Storage(el) for el in json.loads(config.columns)] columns = [Storage(el) for el in json.loads(self.config.columns)]
# check column configuration # check column configuration
# add database field map (tablename, fieldname, keyname) # add database field map (tablename, fieldname, keyname)
...@@ -789,13 +813,15 @@ class Metric1D(List): ...@@ -789,13 +813,15 @@ class Metric1D(List):
A summary information can also be computed for each column or rows. A summary information can also be computed for each column or rows.
Args: Args:
config (gluon.dal.Row): the configuration parameter for the metric. table (gluon.dal.Table): table containing configurations for reports.
selector (EvtSelector): selector handling period of time. id_report (int): identifier of the report in the table.
""" """
def __init__(self, config, selector): def __init__(self, table, id_report):
BaseReport.__init__(self, table, id_report)
BaseReport.__init__(self, config, selector) config = self.config
# group by attributes # group by attributes
field_groupby = config.group_field field_groupby = config.group_field
...@@ -1001,13 +1027,13 @@ class Metric2D(BaseReport): ...@@ -1001,13 +1027,13 @@ class Metric2D(BaseReport):
In fact, all the computation methods of the ``pandas.DataFrame`` class. In fact, all the computation methods of the ``pandas.DataFrame`` class.
Args: Args:
config (gluon.dal.Row): the configuration parameter for the metric. table (gluon.dal.Table): table containing configurations for reports.
selector (EvtSelector): selector handling period of time. id_report (int): identifier of the report in the table.
""" """
def __init__(self, config, selector): def __init__(self, table, id_report):
BaseReport.__init__(self, config, selector) BaseReport.__init__(self, table, id_report)
self._do_metric() self._do_metric()
# replace undefined value by 0 # replace undefined value by 0
......
...@@ -129,7 +129,7 @@ class EvtSelector(SelectorActiveItems): ...@@ -129,7 +129,7 @@ class EvtSelector(SelectorActiveItems):
"""Main selector to build list, metric or graph according to """Main selector to build list, metric or graph according to
the user criteria: the user criteria:
- focus on history record - focus on history record and on a given event.
- Select record active during the selected period of time. - Select record active during the selected period of time.
- Selection can be performed on category defined as a string. - Selection can be performed on category defined as a string.
- For each record computes virtual field: ``age``, ``coverage``, - For each record computes virtual field: ``age``, ``coverage``,
...@@ -144,8 +144,10 @@ class EvtSelector(SelectorActiveItems): ...@@ -144,8 +144,10 @@ class EvtSelector(SelectorActiveItems):
categories, year_end, year_start and year are systematically categories, year_end, year_start and year are systematically
excluded. excluded.
id_event (int): the event identifier
""" """
def __init__(self, table, exclude_fields=()): def __init__(self, table, id_event=0, exclude_fields=()):
li = ["data", li = ["data",
"id_object_categories", "id_object_categories",
...@@ -158,9 +160,10 @@ class EvtSelector(SelectorActiveItems): ...@@ -158,9 +160,10 @@ class EvtSelector(SelectorActiveItems):
li.extend(exclude_fields) li.extend(exclude_fields)
SelectorActiveItems.__init__(self, table, exclude_fields=li) SelectorActiveItems.__init__(self, table, exclude_fields=li)
# add virtual fields
self._db = db = current.globalenv["db"] self._db = db = current.globalenv["db"]
self._id_event = id_event
# add virtual fields
history = db.history history = db.history
virtual = Field.Virtual virtual = Field.Virtual
...@@ -172,13 +175,9 @@ class EvtSelector(SelectorActiveItems): ...@@ -172,13 +175,9 @@ class EvtSelector(SelectorActiveItems):
history.is_over = virtual("is_over", self._is_over, ftype="boolean") history.is_over = virtual("is_over", self._is_over, ftype="boolean")
history.is_start = virtual("is_start", self._is_start, ftype="boolean") history.is_start = virtual("is_start", self._is_start, ftype="boolean")
# keep track of the identifier of the people event
# since it is used by period_activity
self._id_event_people = get_id(db.events, event="People")
def _active_period(self, id_people, id_domain, id_team): def _active_period(self, id_people, id_domain, id_team):
"""Determine the period when a person is active in a given """Determine the period when a person is active in a given
domain and team. domain and team and for a given event
Args: Args:
id_people (int): id_people (int):
...@@ -191,7 +190,7 @@ class EvtSelector(SelectorActiveItems): ...@@ -191,7 +190,7 @@ class EvtSelector(SelectorActiveItems):
stop (datetime or None) stop (datetime or None)
""" """
rawsql = RAWSQL_ACTIVITY % (self._id_event_people, rawsql = RAWSQL_ACTIVITY % (self._id_event,
id_people, id_people,
id_domain, id_domain,
id_team) id_team)
......
...@@ -54,15 +54,21 @@ class SelectorUi(object): ...@@ -54,15 +54,21 @@ class SelectorUi(object):
mdf.configure_field("id_domains", emptyText=text, xtype=mytype) mdf.configure_field("id_domains", emptyText=text, xtype=mytype)
mdf.configure_field("id_fundings", emptyText=text, xtype=mytype) mdf.configure_field("id_fundings", emptyText=text, xtype=mytype)
# to have an unique key use xcomboboxmaster instead of userreset
mdf.configure_field("id_object_categories", mdf.configure_field("id_object_categories",
emptyText=text, emptyText=text,
xtype=mytype) refStore="alias_object_categoriesStore",
userReset=True,
xtype="xcomboboxmaster")
mdf.configure_field("id_object_code", emptyText=text, xtype=mytype) mdf.configure_field("id_object_code", emptyText=text, xtype=mytype)
# to have an unique key use xcomboboxmaster instead of userreset
mdf.configure_field("id_people_categories", mdf.configure_field("id_people_categories",
emptyText=text, emptyText=text,
xtype=mytype) refStore="alias_people_categoriesStore",
userReset=True,
xtype="xcomboboxmaster")
mdf.configure_field("id_people_code", emptyText=text, xtype=mytype) mdf.configure_field("id_people_code", emptyText=text, xtype=mytype)
mdf.configure_field("id_teams", emptyText=text, xtype=mytype) mdf.configure_field("id_teams", emptyText=text, xtype=mytype)
......
# -*- coding: utf-8 -*-
""" NAME
lbfr_people_author_moa
SYNOPSIS
check database consistency between people, author and moa events.
DESCRIPTION
In the track_lhcbfrance application, three events are defined: people
authors and M&O A. The script verify the consistency between them.
OPTIONS
-h, --help
Display the help and exit.
EXAMPLE
> cd ...track_events/scripts
> ./run -S test_lhcbfrance script lbfr_people_author_moa.py
> ./run -S track_lhcbfrance script lbfr_people_author_moa.py
AUTHOR
R. Le Gac -- Jan 2016
"""
from datetime import timedelta
AUTHORS = ['Chercheur',
'Emérite',
'Enseignant-chercheur',
'PhdStudent',
'PostDoc']
MOA = ['Chercheur',
'Emérite',
'Enseignant-chercheur',
'PostDoc']
def check_authors():
"""Check the events consistency for people belonging to the author list.
Author event starts 6 months after the people arrival and ends
one year after its departure.
"""
check_events(get_id(db.projects, project="LHCb"),
get_category_ids(AUTHORS),
"Author",
timedelta(days=182.),
timedelta(days=365))
def check_events(id_project,
people_categorie_ids,
event,
delay_start=timedelta(0.),
delay_end=timedelta(0.)):
"""check the consistency between "people event" and the one selected
in the arguments.
For people having an entry in the "people event" subset, the algorithm
checks that an "event" exits and that its start / end dates agree with
the "people event" ones.
The list of people categories is used to select a subset of
"people events", for example those related to authors, ...
Args:
id_project (int): identifier of the project.
people_categorie_ids (list): list of people categories to build a
subset of "people event".
event (str): the name of the event for which the consistency is checked.
delay_start (timedelta): the delay between the start date of the "people
event" and the "event"
delay_end (timedelta): the delay between the end date of the "people
event" and the "event".
"""
q_cat = db.history.id_people_categories.belongs(people_categorie_ids)
q_project = db.history.id_projects == id_project
id_evt_event = get_id(db.events, event=event)
q_evt_event = (db.history.id_events == id_evt_event) & (q_project)
id_evt_people = get_id(db.events, event="People")
q_evt_people = (db.history.id_events == id_evt_people)
q_evt_people &= (q_cat) & (q_project)
# scan the list of people
for person in db(db.people).select():
id_people = person.id