Merge branch 'master' into 'production'

Release 0.6.4

See merge request !29

controllers/plugin_event.py

@@ -28,25 +28,15 @@ def graph_mpl():
     """Plot list or metric using the matplotlib library
 
     """
-    # selector and configuration
-    try:
-        ui_table = virtdb.selector
-        selector = EvtSelector(ui_table)
-
-    except SelectorActiveItemsException as e:
-        return INLINE_ALERT % (T("Error..."), T(str(e)))
-
-    config = db.graphs[request.vars.id_graphs]
-
     # build the graph
     try:
-        graph = Graph(config, selector)
+        graph = Graph(db.graphs, request.vars.id_graphs)
 
     except (IndexError, TypeError, ValueError) as e:
-        return INLINE_ALERT % ("Error", e)
+        return INLINE_ALERT % ("GRAPH Error", e)
 
     # build the report title
-    title = do_title(config, selector)
+    title = do_title(graph)
 
     # extract the image
     # encode special character to be used in the image URI
@@ -81,39 +71,29 @@ def grid():
     The identifier is used to instantiate the proper class.
 
     """
-    try:
-        ui_table = virtdb.selector
-        selector = EvtSelector(ui_table)
-
-    except SelectorActiveItemsException as e:
-        return INLINE_ALERT % (T("Error..."), T(str(e)))
-
-    # extract the report configuration and the build the report
+    # build the report
     try:
         if "id_lists" in request.vars:
-            config = db.lists[request.vars.id_lists]
-            report = List(config, selector)
+            report = List(db.lists, request.vars.id_lists)
 
         elif "id_metrics1d" in request.vars:
-            config = db.metrics1d[request.vars.id_metrics1d]
-            report = Metric1D(config, selector)
+            report = Metric1D(db.metrics1d, request.vars.id_metrics1d)
 
         elif "id_metrics2d" in request.vars:
-            config = db.metrics2d[request.vars.id_metrics2d]
-            report = Metric2D(config, selector)
+            report = Metric2D(db.metrics2d, request.vars.id_metrics2d)
 
         else:
             return INLINE_ALERT % ("Error", "Report id is not defined.")
 
     except (IndexError, ReportException, TypeError, ValueError) as e:
-        return INLINE_ALERT % ("JSON Error...", e)
+        return INLINE_ALERT % ("REPORT Error...", e)
 
     # extract the configurations for the Ext.data.Store and Dbui.grid.Panel.
     store = report.to_store()
     grid = report.to_grid()
 
     # build the report title
-    title = do_title(config, selector)
+    title = do_title(report)
 
     # delegate the grid rendering to the view
     return dict(cfg_store=store, grid=grid, title=title)

modules/plugin_event/report_objects.py

@@ -13,29 +13,34 @@ from pandas import DataFrame, MultiIndex, to_datetime
 from plugin_dbui import get_id, Store
 from pydal.helpers.methods import smart_query
 from pydal.objects import FieldVirtual
+from selector import EvtSelector
 from StringIO import StringIO
 
 
 MSG_NO_DATAINDEX = "The property dataIndex is required when eval is used."
+MSG_NO_EVT_ID = "Identifier of the event is not defined."
 MSG_NO_XTYPE = "The property xtype is missing."
 
 REG_DBFIELD = re.compile("\w+\.\w+(?:\.\w+)?", re.UNICODE)
+REG_EVT_ID = re.compile("history\.id_events *={1,2} *(\d+)")
 REG_PYQUERY = re.compile("[\( ]*\w+\.\w+\.\w+")
 REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE)
 
 
-def do_title(config, selector):
+def do_title(report):
     """Build the report title.
 
     Args:
-        config (gluon.dal.Row): the list configuration parameter.
-        selector (EvtSelector): selector handling period of time.
+        report (BaseReport): the report
 
     Returns:
         str:
 
     """
-    db = current.globalenv["db"]
+    db = report.db
+
+    config = report.config
+    selector = report.selector
     T = current.T
 
     # from the configuration
@@ -150,40 +155,55 @@ class BaseReport(object):
     """Base class to build list, metric or graph reports.
 
     Args:
-        config (gluon.dal.Row): the configuration parameter for the report.
-        selector (EvtSelector): the selector handling user criteria.
+        table (gluon.dal.Table): table containing configurations for reports.
+        id_report (int): identifier of the report in the table.
 
     """
-    def __init__(self, config, selector):
+    def __init__(self, table, id_report):
 
-        db = current.globalenv["db"]
+        db = table._db
 
         self.db = db
         self.df = None
-        self.config = config
         self.rows = None
-        self.selector = selector
+
+        # Extract the configuration for the report configuration
+        config = table[id_report]
+
+        # Extract the event identifier located in the condition field
+        conditions = config.conditions
+        mtch = REG_EVT_ID.search(conditions)
+        if mtch is None:
+            raise ReportException(current.T(MSG_NO_EVT_ID))
+
+        id_event = int(mtch.group(1))
+
+        # Instantiate the selector
+        virtdb = current.globalenv["virtdb"]
+        selector = EvtSelector(virtdb.selector, id_event)
 
         # apply the condition criteria used to filter the history records
         # condition can be written as a smart query: history.id_events == 7
         # or like a python query: db.events.event == "People"
-        if "conditions" in config:
-            condition = config.conditions
-
-            # minimal protection to avoid injection flow
-            # the beginning of the python query should be like:
-            #    db.table.field
-            #    (db.table.field
-            #    ((db.table.field
-            #    ( ( db.table.field
-            #
-            if REG_PYQUERY.match(condition):
-                q_conditions = eval(condition, None, {"db": db})
 
-            else:
-                q_conditions = smart_query(db.history, config.conditions)
+        # minimal protection to avoid injection flow
+        # the beginning of the python query should be like:
+        #    db.table.field
+        #    (db.table.field
+        #    ((db.table.field
+        #    ( ( db.table.field
+        #
+        if REG_PYQUERY.match(conditions):
+            q_conditions = eval(conditions, None, {"db": db})
 
-            selector.append_query(q_conditions)
+        else:
+            q_conditions = smart_query(db.history, conditions)
+
+        selector.append_query(q_conditions)
+
+        # keep track of configuration and selector
+        self.config = config
+        self.selector = selector
 
     def _do_data(self, maps):
         """Build a temporarily list with the raw data for each series.
@@ -274,19 +294,23 @@ class Graph(BaseReport):
     scater plots, *etc*.
 
     Args:
-        config (gluon.dal.Row):
-            the configuration parameter for the graph.
+        table (gluon.dal.Table):
+            table containing configurations for reports.
 
-        selector (EvtSelector):
-            selector handling period of time.
+        id_report (int):
+            identifier of the report in the table.
 
         backend (str):
             the name of the matplotlib backend uses to produce figure.
 
     """
-    def __init__(self, config, selector, backend="Agg"):
+    def __init__(self, table, id_report, backend="Agg"):
 
-        BaseReport.__init__(self, config, selector)
+        self.db = table._db
+
+        self.config = config = table[id_report]
+        self.df = None
+        self.rows = None
 
         # set the matplotlib back end
         #
@@ -315,18 +339,18 @@ class Graph(BaseReport):
         report_type = config.report_type
         report_name = config.report_name
         report_id = get_id(db[report_type], name=report_name)
-        report_config = db[report_type][report_id]
 
         if report_type == "lists":
-            report = List(report_config, selector)
+            report = List(db.lists, report_id)
 
         elif report_type == "metrics1d":
-            report = Metric1D(report_config, selector)
+            report = Metric1D(db.metrics1d, report_id)
 
         elif report_type == "metrics2d":
-            report = Metric2D(report_config, selector)
+            report = Metric2D(db.metrics2d, report_id)
 
         self.df = report.to_df()
+        self.selector = report.selector
 
         # build the graph from the DataFrame
         self._do_graph()
@@ -458,16 +482,16 @@ class List(BaseReport):
     Its configuration is returned by the method *to_store*.
 
     Args:
-        config (gluon.dal.Row): the configuration parameter for the list.
-        selector (EvtSelector): selector handling period of time.
+        table (gluon.dal.Table): table containing configurations for reports.
+        id_report (int): identifier of the report in the table.
 
     """
-    def __init__(self, config, selector):
+    def __init__(self, table, id_report):
 
-        BaseReport.__init__(self, config, selector)
+        BaseReport.__init__(self, table, id_report)
 
         # decode column configuration
-        columns = [Storage(el) for el in json.loads(config.columns)]
+        columns = [Storage(el) for el in json.loads(self.config.columns)]
 
         # check column configuration
         # add database field map (tablename, fieldname, keyname)
@@ -789,13 +813,15 @@ class Metric1D(List):
     A summary information can also be computed for each column or rows.
 
     Args:
-        config (gluon.dal.Row): the configuration parameter for the metric.
-        selector (EvtSelector): selector handling period of time.
+        table (gluon.dal.Table): table containing configurations for reports.
+        id_report (int): identifier of the report in the table.
 
     """
-    def __init__(self, config, selector):
+    def __init__(self, table, id_report):
 
-        BaseReport.__init__(self, config, selector)
+        BaseReport.__init__(self, table, id_report)
+
+        config = self.config
 
         # group by attributes
         field_groupby = config.group_field
@@ -1001,13 +1027,13 @@ class Metric2D(BaseReport):
     In fact, all the computation methods of the ``pandas.DataFrame`` class.
 
     Args:
-        config (gluon.dal.Row): the configuration parameter for the metric.
-        selector (EvtSelector): selector handling period of time.
+        table (gluon.dal.Table): table containing configurations for reports.
+        id_report (int): identifier of the report in the table.
 
     """
-    def __init__(self, config, selector):
+    def __init__(self, table, id_report):
 
-        BaseReport.__init__(self, config, selector)
+        BaseReport.__init__(self, table, id_report)
         self._do_metric()
 
         # replace undefined value by 0

modules/plugin_event/selector.py

@@ -129,7 +129,7 @@ class EvtSelector(SelectorActiveItems):
     """Main selector to build list, metric or graph according to
     the user criteria:
 
-        - focus on history record
+        - focus on history record and on a given event.
         - Select record active during the selected period of time.
         - Selection can be performed on category defined as a string.
         - For each record computes virtual field: ``age``, ``coverage``,
@@ -144,8 +144,10 @@ class EvtSelector(SelectorActiveItems):
             categories, year_end, year_start and  year are systematically
             excluded.
 
+        id_event (int): the event identifier
+
     """
-    def __init__(self, table, exclude_fields=()):
+    def __init__(self, table, id_event=0, exclude_fields=()):
 
         li = ["data",
               "id_object_categories",
@@ -158,9 +160,10 @@ class EvtSelector(SelectorActiveItems):
         li.extend(exclude_fields)
         SelectorActiveItems.__init__(self, table, exclude_fields=li)
 
-        # add virtual fields
         self._db = db = current.globalenv["db"]
+        self._id_event = id_event
 
+        # add virtual fields
         history = db.history
         virtual = Field.Virtual
 
@@ -172,13 +175,9 @@ class EvtSelector(SelectorActiveItems):
         history.is_over = virtual("is_over", self._is_over, ftype="boolean")
         history.is_start = virtual("is_start", self._is_start, ftype="boolean")
 
-        # keep track of the identifier of the people event
-        # since it is used by period_activity
-        self._id_event_people = get_id(db.events, event="People")
-
     def _active_period(self, id_people, id_domain, id_team):
         """Determine the period when a person is active in a given
-        domain and team.
+        domain and team and for a given event
 
         Args:
             id_people (int):
@@ -191,7 +190,7 @@ class EvtSelector(SelectorActiveItems):
                 stop (datetime or None)
 
         """
-        rawsql = RAWSQL_ACTIVITY % (self._id_event_people,
+        rawsql = RAWSQL_ACTIVITY % (self._id_event,
                                     id_people,
                                     id_domain,
                                     id_team)

modules/plugin_event/ui_selector.py

@@ -54,15 +54,21 @@ class SelectorUi(object):
         mdf.configure_field("id_domains", emptyText=text, xtype=mytype)
         mdf.configure_field("id_fundings", emptyText=text, xtype=mytype)
 
+        # to have an unique key use xcomboboxmaster instead of userreset
         mdf.configure_field("id_object_categories",
                             emptyText=text,
-                            xtype=mytype)
+                            refStore="alias_object_categoriesStore",
+                            userReset=True,
+                            xtype="xcomboboxmaster")
 
         mdf.configure_field("id_object_code", emptyText=text, xtype=mytype)
 
+        # to have an unique key use xcomboboxmaster instead of userreset
         mdf.configure_field("id_people_categories",
                             emptyText=text,
-                            xtype=mytype)
+                            refStore="alias_people_categoriesStore",
+                            userReset=True,
+                            xtype="xcomboboxmaster")
 
         mdf.configure_field("id_people_code", emptyText=text, xtype=mytype)
         mdf.configure_field("id_teams", emptyText=text, xtype=mytype)

scripts/lbfr_people_author_moa.py

-# -*- coding: utf-8 -*-
-""" NAME
-        lbfr_people_author_moa
-
-    SYNOPSIS
-        check database consistency between people, author and moa events.
-
-    DESCRIPTION
-        In the track_lhcbfrance application, three events are defined: people
-        authors and M&O A. The script verify the consistency between them.
-
-    OPTIONS
-
-        -h, --help
-            Display the help and exit.
-
-    EXAMPLE
-
-        > cd ...track_events/scripts
-        > ./run -S test_lhcbfrance script lbfr_people_author_moa.py
-        > ./run -S track_lhcbfrance script lbfr_people_author_moa.py
-
-    AUTHOR
-        R. Le Gac -- Jan 2016
-
-"""
-from datetime import timedelta
-
-
-AUTHORS = ['Chercheur',
-           'Emérite',
-           'Enseignant-chercheur',
-           'PhdStudent',
-           'PostDoc']
-
-MOA = ['Chercheur',
-       'Emérite',
-       'Enseignant-chercheur',
-       'PostDoc']
-
-
-def check_authors():
-    """Check the events consistency for people belonging to the author list.
-
-    Author event starts 6 months after the people arrival and ends
-    one year after its departure.
-
-    """
-    check_events(get_id(db.projects, project="LHCb"),
-                 get_category_ids(AUTHORS),
-                 "Author",
-                 timedelta(days=182.),
-                 timedelta(days=365))
-
-
-def check_events(id_project,
-                 people_categorie_ids,
-                 event,
-                 delay_start=timedelta(0.),
-                 delay_end=timedelta(0.)):
-    """check the consistency between "people event" and the one selected
-    in the arguments.
-
-    For people having an entry in the "people event" subset, the algorithm
-    checks that an "event" exits and that its start / end dates agree with
-    the "people event" ones.
-
-    The list of people categories is used to select a subset of
-    "people events", for example those related to authors, ...
-
-        Args:
-        id_project (int): identifier of the project.
-        people_categorie_ids (list): list of people categories to build a
-            subset of "people event".
-        event (str): the name of the event for which the consistency is checked.
-        delay_start (timedelta): the delay between the start date of the "people
-            event" and the "event"
-        delay_end (timedelta): the delay between the end date of the "people
-            event" and the "event".
-
-    """
-    q_cat = db.history.id_people_categories.belongs(people_categorie_ids)
-    q_project = db.history.id_projects == id_project
-
-    id_evt_event = get_id(db.events, event=event)
-    q_evt_event = (db.history.id_events == id_evt_event) & (q_project)
-
-    id_evt_people = get_id(db.events, event="People")
-    q_evt_people = (db.history.id_events == id_evt_people)
-    q_evt_people &= (q_cat) & (q_project)
-
-    # scan the list of people
-    for person in db(db.people).select():
-
-        id_people = person.id
-        if id_people == UNDEF_ID:
-            continue
-
-        last_name = db.people[id_people].last_name
-
-        # look for "people event(s)" associates to the person
-        # determine start and end dates of people event
-        tpl = evt_people(q_evt_people, id_people)
-        if len(tpl) == 0:
-            continue
-
-        start_date, end_date, team_ids = tpl
-
-        # computed the expected dates for the event "event"
-        exp_start = (None if start_date is None else start_date + delay_start)
-        exp_end = (None if end_date is None else end_date + delay_end)
-
-        # check the event "event"
-        query = (q_evt_event) & (db.history.id_people == id_people)
-        myset = db(query)
-        nrows = myset.count()
-
-        # signal too many event
-        if nrows > 1:
-            print last_name, start_date, end_date, "→ to many %s event!\n" % event
-            print "Have a look and fix it by hand.\n"
-            continue
-
-        # the event is missing, add it
-        if  nrows == 0:
-            print last_name, start_date, end_date, '→ no %s event' % event
-
-            if len(team_ids) != 1:
-                print "Team change during the period."
-                print "Have a look and fix it by hand.\n"
-                continue
-
-            data = dict(id_events=id_evt_event,
-                        id_people=id_people,
-                        id_projects=id_project,
-                        id_teams=list(team_ids)[0],
-                        start_date= exp_start,
-                        end_date=exp_end)
-
-            fix_event(**data)
-            continue
-
-        # the event exits
-        row = myset.select(db.history.ALL).first()
-        evt_start, evt_end = row.start_date, row.end_date
-
-        # check the start date
-        # deal with special case
-        ok_start = evt_start is None and start_date is None
-        ok_start |= evt_start <= start_date
-        ok_start |= evt_start == exp_start
-
-        if not ok_start:
-            print last_name, "→ wrong %s start date" % event
-            print "The person start  :", start_date
-            print "The event start   :", evt_start
-            print "The expected start:", exp_start
-            if evt_start and exp_start:
-                print "Delta [days]      :", (evt_start - exp_start).days
-
-            data = dict(id=row.id, start_date= exp_start)
-            fix_event(**data)
-
-        # check the end date
-        ok_end = evt_end is None and end_date is None
-        ok_end |= evt_end == exp_end
-
-        if not ok_end:
-            print last_name, "→ wrong %s end date" % event
-            print "The person quit :", end_date
-            print "The event end   :", evt_end
-            print "The expected end:", exp_end
-            if evt_end and exp_end:
-                print "Delta [days]    :", (evt_end - exp_end).days
-
-            data = dict(id=row.id, end_date= exp_end)
-            fix_event(**data)
-
-
-def check_moa():
-    """Check the events consistency for people belonging to the M&O A list.
-
-    """
-    check_events(get_id(db.projects, project="LHCb"),
-                 get_category_ids(MOA),
-                 "M&O Cat A",
-                 timedelta(days=0),
-                 timedelta(days=0))
-
-
-def evt_people(query_event_people, id_people):
-    """For a given person determine the minimum start date and the
-    maximum end date of the people event. It also extract the list
-    of team ids.
-
-    Args:
-        query_event_people(gluon.dal.Query): the query to select all the
-            "people event" or a subset of them.
-        id_people (int): the identifier of the person in the people table.
-
-    Returns:
-        tuple: containing the start and the end dates. It is empty
-            when there is no people events associated to the person.
-            The last element is the list of id_teams.
-    """
-    start_dates, end_dates, team_ids = set(), set(), set()
-
-    query = (query_event_people) & (db.history.id_people == id_people)
-    myset = db(query)
-    if myset.count() == 0:
-        return ()
-
-    for row in myset.select(db.history.ALL):
-        start_dates.add(row.start_date)
-        end_dates.add(row.end_date)
-        team_ids.add(row.id_teams)
-
-    start_date = (None if None in start_dates else min(start_dates))
-    end_date = (None if None in end_dates else max(end_dates))
-
-    return (start_date, end_date, team_ids)
-
-
-def fix_event(**kwargs):
-    """Fix the event.
-
-    Insert of update the event.
-
-    Keyword arguments:
-        id (int): id of the event