Commit f540b4d2 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Polish the report_object classes making Metric1D working with merged events.

parent a15174c4
......@@ -7,7 +7,12 @@ import re
from plugin_dbui import INLINE_ALERT
from report_objects import do_title, Graph, List, Metric1D, Metric2D
from report_objects import (do_title,
Graph,
List,
Metric1D,
Metric2D,
ReportException)
from selector import MySelector, SelectorActiveItemsException
......@@ -109,7 +114,7 @@ def grid():
else:
return INLINE_ALERT % ("Error", "Report id is not defined.")
except (IndexError, TypeError, ValueError) as e:
except (IndexError, ReportException, TypeError, ValueError) as e:
return INLINE_ALERT % ("JSON Error...", e)
# extract the configurations for the Ext.data.Store and App.grid.Panel.
......
......@@ -320,7 +320,6 @@
'Tables': 'Tables',
'Team': 'Équipe',
'teams': 'équipes',
'Tests': 'Tests',
'The aggregation function applies on the metric field. It is computed on the subset of row having a given values for the group_field x and y.': "La function d'agrégation est appliquée sur les valeurs du champ métrique. Elle est calculée sur le sous-ensemble d'enregisteremnts ayant une certain valeurs pour les champs 'group_field' x et y.",
'the graphs': 'les graphiques',
'The graphs': 'Les graphiques',
......@@ -331,6 +330,8 @@
'the metrics 1d': 'les métriques 1d',
'the metrics 2d': 'les métriques 2d',
'The metrics 2d': 'Les métriques 2d',
'The property dataIndex is required when eval is used.': 'La propriété dataIndex doit être définie quand eval est utilisé.',
'The property xtype is missing.': 'La propriété xtype doit être définie.',
"The report id '%s' is unknown.": "The report id '%s' is unknown.",
'This email already has an account': 'This email already has an account',
'Time': 'Temps',
......
......@@ -15,7 +15,8 @@ def_columns = \
"align": "right",
"dbfield": "?.?",
"format": "0.0",
"text": "?"
"text": "?",
"xtype": "numbercolumn"
}]"""
......
......@@ -14,6 +14,8 @@ from gluon.storage import Storage
from plugin_dbui import get_id, Store
from StringIO import StringIO
MSG_NO_DATAINDEX = "The property dataIndex is required when eval is used."
MSG_NO_XTYPE = "The property xtype is missing."
REG_DBFIELD = re.compile("\w+\.\w+(?:\.\w+)?", re.UNICODE)
REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE)
......@@ -445,12 +447,11 @@ class List(BaseReport):
# decode column configuration
columns = [Storage(el) for el in json.loads(config.columns)]
# Add database field map (tablename, fieldname, keyname)
map(self._add_map, columns)
# check column configuration
# add database field map (tablename, fieldname, keyname)
# add the dataIndex (DataFrame, Ext.data.Store, Ext.grid.Panel)
map(self._add_dataIndex, columns)
map(self._check_column, columns)
# columns are persistent
self._columns = columns
......@@ -459,36 +460,12 @@ class List(BaseReport):
self._do_metric()
def _add_map(self, column):
"""Helper function to add the database field map
to the column configuration.
@type column: gluon.storage.Storage
@param column:
"""
if column.dbfield:
column.map = split_dbfield(column.dbfield)
def _add_dataIndex(self, column):
"""Helper function to add the dataIndex to the column configuration
when it is not defined. It is equal to the C{dbfield} property in
which dots are removed
@type column: gluon.storage.Storage
@param column:
"""
if not (column.dataIndex or column.xtype == "rownumberer"):
column.dataIndex = column.dbfield.replace('.', '')
def _cast_to_xtype(self, column, xtype):
"""Cast the type of a dataframe column to the column xtype.
def _cast_type(self, column, map, xtype):
"""Cast the type of a dataframe column to the database field type
or to the grid column xtype.
The type of the column determine by the pandas might be wrong.
This append when value are undefined or missing.
This append when events are merged with different user block.
This is fine in most of the case but not with computed column.
In that case the eval computation crashed.
......@@ -499,18 +476,45 @@ class List(BaseReport):
@type column: str
@param column: the index of the column in the DataFrame
@type map: tuple
@param map: address of the database field encoded as
(tablename, fieldname, keyname).
@type xtype: str
@param xtype: the xtype of the grid column.
Possible values are 'booleancolumn, datecolumn, gridcolumn
Possible values are booleancolumn, datecolumn, gridcolumn
and numbercolumn.
"""
df = self.df
tablename, fieldname, keyname = map
# the dtype of column containing a mixture of type is object.
if (tablename == 'year') or (df[column].dtype != 'object'):
return
dbtype = self.db[tablename][fieldname].type
if df[column].dtype == 'object' and xtype != None:
# the dtype for column containing string is also object
if dbtype in ('string', 'text'):
return
elif dbtype == 'boolean':
df[column] = df[column].astype('bool')
elif dbtype in ('date', 'datetime', 'time'):
df[column] = pd.to_datetime(df[column])
elif dbtype in ('double', 'integer'):
df[column] = df[column].astype('float64')
# database field containing JSON-type dictionary
# The type of the key is defined in the event model but it is
# not accessible at this stage. Instead we use the grid column xtype.
elif dbtype == 'json':
if xtype == 'gridcolumn':
return
pass
elif xtype == 'booleancolumn':
df[column] = df[column].astype('bool')
......@@ -522,6 +526,35 @@ class List(BaseReport):
df[column] = df[column].astype('float64')
def _check_column(self, column):
"""Check column configuration:
- Raise an exception if xtype is not defined
- Raise an exception when eval is defined but not the dataIndex
- Add the database field map
- Add the dataIndex if not defined
@type column: gluon.storage.Storage
@param column:
"""
T = current.T
xtype = column.xtype
if not xtype:
raise ReportException(T(MSG_NO_XTYPE))
if column.eval and not column.dataIndex:
raise ReportException(T(MSG_NO_DATAINDEX))
dbfield = column.dbfield
if dbfield:
column.map = split_dbfield(dbfield)
if not (column.dataIndex or xtype == "rownumberer"):
column.dataIndex = column.dbfield.replace('.', '')
def _do_metric(self):
"""Interface the database with the DataFrame structure.
This method handle the "year" database field.
......@@ -552,8 +585,8 @@ class List(BaseReport):
# make the data frame persistent
self.df = df
# cast dataframe column type to grid column xtype
map(self._cast_to_xtype, index, xtypes)
# cast dataframe column type to database type or grid column xtype
map(self._cast_type, index, maps, xtypes)
# add computed columns
for el in columns:
......@@ -618,9 +651,12 @@ class List(BaseReport):
dbfield = db[tablename][fieldname]
cfg.type = dbfield.type
if dbfield.type in ('string', 'text', 'json'):
if dbfield.type in ('blob', 'string', 'text', 'json'):
cfg.type = 'string'
elif dbfield.type == 'boolean':
cfg.type = 'boolean'
elif dbfield.type in ('date', 'datetime', 'time'):
cfg.type = 'date'
cfg.dateFormat = 'c'
......@@ -693,7 +729,7 @@ class List(BaseReport):
return self._store
class Metric1D(BaseReport):
class Metric1D(List):
"""A Metric1D is a table displaying metrics when records are
group by value for a given database field. for example, the C{cell[i]}
of the C{table.field2} column contains the sum
......@@ -721,19 +757,28 @@ class Metric1D(BaseReport):
# group by attributes
field_groupby = config.group_field
if config.group_field == "year":
if field_groupby == "year":
index_groupby = 'year'
text_groupby = 'year'
else:
tu = split_dbfield(config.group_field)
index_groupby = (tu[2] if tu[2] else tu[1])
# decode column configuration
columns = [Storage(dbfield=field_groupby, text=index_groupby, aggregate="")]
tu = split_dbfield(field_groupby)
index_groupby = field_groupby.replace('.', '')
text_groupby = (tu[2] if tu[2] else tu[1])
# first column contains group by information
first_column = Storage(aggregate="",
dbfield=field_groupby,
text=text_groupby,
xtype='gridcolumn')
# columns configuration
columns = [first_column]
columns.extend([Storage(el) for el in json.loads(config.columns)])
# add database field map and dataIndex property
map(self._add_map, columns)
map(self._add_dataIndex, columns)
# check column configuration
# add database field map (tablename, fieldname, keyname)
# add the dataIndex (DataFrame, Ext.data.Store, Ext.grid.Panel)
map(self._check_column, columns)
# persistence
self._columns = columns
......@@ -743,30 +788,6 @@ class Metric1D(BaseReport):
self._do_metric()
def _add_map(self, column):
"""Helper function to add the database field map
to the column configuration.
@type column: gluon.storage.Storage
@param column:
"""
if column.dbfield:
column.map = split_dbfield(column.dbfield)
def _add_dataIndex(self, column):
"""Helper function to add the dataIndex to the column configuration
when it is not defined. It is equal to the C{text} property.
@type column: gluon.storage.Storage
@param column:
"""
if not (column.dataIndex or column.xtype == "rownumberer"):
column.dataIndex = column.text
def _do_metric(self):
"""Compute the metric according to user specifications.
......@@ -778,18 +799,22 @@ class Metric1D(BaseReport):
field_groupby = self._field_groupby
index_groupby = self._index_groupby
# extract columns associated to database field
cfg = [el for el in columns if el.dbfield]
indexes = [el.dataIndex for el in cfg]
maps = [el.map for el in cfg if el.dbfield]
# extract aggregation operator for each column
operators = dict()
for i in xrange(len(cfg)):
el = cfg[i]
if el.aggregate:
operators[indexes[i]] = el.aggregate
# extract columns associated to database field
# and their aggregation function
indexes, maps, operators, xtypes = [], [], {}, []
for column in columns:
if column.dbfield:
dataIndex = column.dataIndex
indexes.append(dataIndex)
maps.append(column.map)
aggregate = column.aggregate
if aggregate:
operators[dataIndex] = aggregate
xtypes.append(column.xtype)
# interrogate the database and fill the data frame
data = self._do_data(maps)
......@@ -800,7 +825,11 @@ class Metric1D(BaseReport):
return
df = pd.DataFrame(data, columns=indexes)
# cast dataframe column type to database field or grid column types
self.df = df
map(self._cast_type, indexes, maps, xtypes)
# aggregate the data running dedicated operator for each column
df = df.groupby(index_groupby)
df = df.agg(operators)
......@@ -819,7 +848,7 @@ class Metric1D(BaseReport):
if isinstance(df.index, pd.core.index.MultiIndex):
df.index = [el[1] for el in df.index]
# the data frame is persistent
# make the data frame persistent
self.df = df
......@@ -839,15 +868,13 @@ class Metric1D(BaseReport):
The key is equal to the C{Ext.data.Field} name.
"""
df = self.df
data = df.to_dict(orient='records')
# add the groupby value to each row
index_groupby = self._index_groupby
map(lambda row, v: row.__setitem__(index_groupby, v), data, df.index)
# build the output: a list of dictionary
# The latter contains the key group, count, avg_age and sum_fte
data = []
for group, di in self.df.T.to_dict().iteritems():
di[index_groupby] = group
data.append(di)
self._store.data.extend(data)
......@@ -857,17 +884,28 @@ class Metric1D(BaseReport):
and data are grouped per the group field values.
"""
columns = self._columns
index_groupby = self._index_groupby
store = self._store
# firs column is for the group field
store.fields = [dict(name=index_groupby, type='string')]
# add the others one
for el in self.df.columns:
di = dict(name=el, type='float')
store.fields.append(di)
for column in columns:
cfg = Storage(name=column.dataIndex)
xtype = column.xtype
if xtype == 'numbercolumn':
cfg.type = 'float'
elif xtype == 'booleancolumn':
cfg.type = 'boolean'
elif xtype == 'datecolumn':
cfg.type = 'date'
elif xtype == 'gridcolumn':
cfg.type = 'string'
store.fields.append(cfg)
store.sorters = [index_groupby]
......@@ -884,23 +922,14 @@ class Metric1D(BaseReport):
grid = Storage(columns=[], features=[])
# first column is for the group field
grid.columns = [{'text': current.T(index_groupby.title()),
'dataIndex': index_groupby,
'columnWidth': 0.8}]
# the other are already specified by the user
for cfg in columns[1:]:
# grid column configuration
for cfg in columns:
# remove non Ext JS property
for key in ('aggregate', 'dbfield', 'eval', 'map'):
if key in cfg:
del cfg[key]
# column type
if 'xtype' not in cfg:
cfg['xtype'] = 'numbercolumn'
grid.columns.append(cfg)
# activate summary feature
......
--------------------------------- CHANGE LOG ----------------------------------
HEAD
- List, Metric1D and 2D works with missing data.
It appends when several events are selected at at the same time.
- Re-enforce column configuration rules: xtype has always to be defined,
dataIndex is required with eval.
0.4.5 (Mar 2015)
- Major release non-backward compatible.
- Migrate to plugin_dbui 0.6.2.6, pluggin_ace 1.1.8
- Migrate to plugin_dbui 0.6.2.6, pluggin_ace 1.1.8
and to python-pandas 0.15.2
- Migrate from sqlite to mariadb.
- Migrate to matplotlib 1.3.1 instead of Ext.chart.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment