Commit 9b29d244 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

List can be sort using DESC criteria.

parent 6c202ee3
......@@ -23,25 +23,25 @@ REG_SINGLE_DBFIELD = re.compile("^ *\w+\.\w+(\.\w+)? *$", re.UNICODE)
def do_title(config, selector):
"""Build the report title.
"""
db = current.globalenv['db']
T = current.T
# from the configuration
title = (config.title if config.title else config.name)
# add meta data
metadata = []
if selector.id_teams:
metadata.append(db.teams[selector.id_teams].team)
if selector.id_projects:
metadata.append(db.projects[selector.id_projects].project)
if selector.category:
metadata.append(selector.category)
if selector.id_people_categories:
code = db.people_categories[selector.id_people_categories].code
metadata.append(code)
......@@ -49,52 +49,52 @@ def do_title(config, selector):
# add period
if selector.year_start and not selector.year_end:
metadata.append(str(selector.year_start))
elif selector.year_start and selector.year_end:
years = (str(selector.year_start), str(selector.year_end))
metadata.append(T("from %s to %s") % years)
return "%s: %s" % (title, ' / '.join(metadata))
def get_value(row, tablename, fieldname, keyname='', **kwargs):
"""Return the row value of the database field
identify by the tablename, fieldname and keyname
@type row: gluon.dal.Row
@param row:
@type tablename: str
@param tablename:
@type fieldname: str
@param fieldname:
@type keyname: str
@param keyname:
@return:
@return:
- C{row[tablename][fieldname]} or C{row[fieldname]}
when tablename and fieldname are defined
when tablename and fieldname are defined
- C{row[tablename][fieldname][keyname]} for JSON type field
- C{kwargs[tablename]} when fieldname and keyname are not defined
- None when the field address does not exist in the row
"""
undefined = None
# force value
if tablename and (not fieldname) and (tablename in kwargs):
return kwargs[tablename]
# field is addressed in the row by tablename and by fieldname
if tablename in row:
if tablename in row:
value = row[tablename][fieldname]
# field is addressed in the row by its fieldname only
elif fieldname in row:
value = row[fieldname]
else:
return undefined
......@@ -102,10 +102,10 @@ def get_value(row, tablename, fieldname, keyname='', **kwargs):
# it has been design for JSON-type field containing a dictionary
if not keyname:
return value
elif keyname and keyname in value:
return value[keyname]
return undefined
......@@ -114,15 +114,15 @@ def split_dbfield(value):
as C{table.field} or C{table.field.key}. The latter is used for
JSON type field. It is decode as a 3-elements tuple in
which the last element is equal to key or an empty string.
@type value: string
@param value: the address of the database field, either
tablename.fieldname or tablename.fieldname.key
@rtype: tuple
@return: the tuple C{(tablename, fieldname, keyname)}.
The C{keyname} is either a string or an empty string.
"""
li = value.split('.')
if len(li) == 1:
......@@ -137,47 +137,47 @@ class ReportException(BaseException): pass
class BaseReport(object):
"""Base class for list, metric and graph reports.
@type db: gluon.dal.DAL
@ivar db: the database connection
@type df: DataFrame or None
@ivar df: The data frame containing the data form well-formed
@ivar df: The data frame containing the data form well-formed
for the report.
@type config: gluon.dal.Row
@ivar config: the report configuration extracts from the
database.
@type rows: gluon.dal.Rows or None
@ivar rows: the record selected from the history table.
@type selector: MySelector
@ivar selector: the selector containing user criteria
"""
def __init__(self, config, selector):
"""
@type config: gluon.dal.Row
@param config: the list configuration parameter
@type selector: MySelector
@param selector: the selector handling user criteria
"""
db = current.globalenv['db']
self.db = db
self.df = None
self.config = config
self.rows = None
self.selector = selector
# apply the condition criteria used to filter the history records
if "conditions" in config:
q_conditions = smart_query(db.history, config.conditions)
selector.append_query(q_conditions)
def _do_data(self, maps):
"""Build a temporarily list with the raw data for each series.
......@@ -186,12 +186,12 @@ class BaseReport(object):
@type maps: list
@param maps: the database field map (tablename, fieldname, keyname).
One per series.
@rtype: list
"""
data = []
config = self.config
db = self.db
selector = self.selector
......@@ -201,69 +201,69 @@ class BaseReport(object):
# limit the list of database fields to speed up processing:
# - keep those required by the user
# - remove virtual field
# - add standard fields require to compute virtual fields
# - add standard fields require to compute virtual fields
dbfields = [db[el[0]][el[1]] for el in maps if el[1]]
dbfields = [el for el in dbfields if not isinstance(el, FieldVirtual)]
dbfields.extend([db.history.end_date,
dbfields.extend([db.history.end_date,
db.history.percentage,
db.history.start_date,
db.history.start_date,
db.people.birth_date])
# the year axis is on
# scan the database and compute virtual field on the year basis
# scan the database and compute virtual field on the year basis
if self._is_year(maps):
# get the year range
for year in selector.get_years():
selector.set_year(year)
for row in db(query(db.history)).select(*dbfields):
values = [get_value(row, *map, year=year) for map in maps]
data.append(values)
# standard scan
else:
for row in db(query(db.history)).select(*dbfields):
for row in db(query(db.history)).select(*dbfields):
values = [get_value(row, *map) for map in maps]
data.append(values)
return data
def _is_year(self, maps):
"""
@type maps: list
@param maps: the database field map (tablename, fieldname, keyname).
One per series.
@rtype: boolean
@return: C{True} is the pseudo field C{year} is in maps
"""
li = [True for el in maps if el[0] == 'year']
return (True if li else False)
return (True if li else False)
def to_df(self):
"""Return the pandas DataFrame.
@rtype: pandas.DataFrame
"""
return self.df
class Graph(BaseReport):
"""Any data encapsulated in list, 1-dim or 2-dim metrics
can be displayed as a graph. The rendering is performed by
the matplotlib library. Therefore, many representations of
the data are possible: plot, histogram, bar charts, errorcharts,
"""Any data encapsulated in list, 1-dim or 2-dim metrics
can be displayed as a graph. The rendering is performed by
the matplotlib library. Therefore, many representations of
the data are possible: plot, histogram, bar charts, errorcharts,
scaterplots, etc.
@type ax: matplotlib.AxesSubplot
@ivar ax:
"""
def __init__(self, config, selector, backend="Agg"):
......@@ -273,23 +273,23 @@ class Graph(BaseReport):
#
# NOTE: the X11 back end is not needed on the server side. In addition
# Tkinter crash with the message "out of stack space" when the 2nd plot
# is generated.
# is generated.
# The documentation recommend to limit the matplotlib back end to Agg
# which is tuned to render high quality PNG image. But, it is also
# design to render PDF and SVG image without the X11 interface.
#
#
matplotlib.use(backend)
# split the plot configuration in two parts:
# 1) keywords for the DataFrame.plot method
# 2) steering parameter for this class
config.plot = json.loads(config.plot)
config.steer = Storage()
for k in ('index', 'transpose', 'xlabel', 'ylabel'):
v = config.plot.pop(k, None)
config.steer[k] = v
# instantiate the DataFrame for the report
db = self.db
......@@ -300,146 +300,146 @@ class Graph(BaseReport):
if report_type == "lists":
report = List(report_config, selector)
elif report_type == "metrics1d":
report = Metric1D(report_config, selector)
elif report_type == "metrics2d":
report = Metric2D(report_config, selector)
self.df = report.to_df()
# build the graph from the DataFrame
self._do_graph()
self._do_labels()
self._do_legend()
self._do_tick()
def _do_graph(self):
"""Build the graph from the C{DataFrame} structure.
"""
config = self.config
df = self.df
plot, steer = config.plot, config.steer
# transpose
if steer.transpose:
df = df.T
# generate the plot using a specific set of columns
if steer.index and len(steer.index) <= len(df.columns):
ax = df.ix[:,steer.index].plot(**plot)
# generate the plot using all columns
else:
ax = df.ix[:,:].plot(**plot)
# persistence
self.ax = ax
def _do_labels(self):
"""Deal with axes label.
"""
ax = self.ax
steer = self.config.steer
if steer.xlabel:
ax.set_xlabel(steer.xlabel, x=1, horizontalalignment='right')
if steer.ylabel:
ax.set_ylabel(steer.ylabel, y=1, horizontalalignment='right')
def _do_legend(self):
"""Deal with legend.
"""
ax = self.ax
if ax.get_legend():
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width, box.height * 0.9])
ax.legend(loc='lower right',
bbox_to_anchor=(1.01, 1.),
ax.legend(loc='lower right',
bbox_to_anchor=(1.01, 1.),
fontsize=10,
ncol=3)
def _do_tick(self):
"""Polish the tick mark
"""
ax = self.ax
ax.minorticks_on()
ax.tick_params(which='major', length=8)
ax.tick_params(which='minor', length=4)
def _savefig(self, format):
"""Save the figure as a string.
@type format: str
@param format: possible values are pdf, png and svg.
"""
fig = self.ax.get_figure()
fi = StringIO()
fig.savefig(fi, format=format)
data = fi.getvalue()
fi.close()
fig.clear()
matplotlib.pyplot.close(fig)
return data
def to_pdf(self):
"""
@rtype: string
@return: encode the graph with the PDF format.
"""
return self._savefig('pdf')
return self._savefig('pdf')
def to_png(self):
"""
@rtype: string
@return: encode the graph with the PNG format.
"""
return self._savefig('png')
def to_svg(self):
"""
@rtype: string
@return: encode the graph with the SVG format.
"""
return self._savefig('svg')
class List(BaseReport):
"""A list is a table in which each column contains the values of
one database field. The rows can be grouped per value of a given column.
Summary information can be computed for each group as well as for
the whole table.
"""A list is a table in which each column contains the values of
one database field. The rows can be grouped per value of a given column.
Summary information can be computed for each group as well as for
the whole table.
The list is displayed as the C{App.grid.Panel} widget.
The configuration of the list columns is the configuration of
the C{App.grid.Panel} widget.
More technically, this class interfaces the database and the
C{App.grid.Panel} thought the underlying C{Ext.data.Store}.
Its configuration is returned by the method L{to_store}.
"""
def __init__(self, config, selector):
......@@ -448,66 +448,66 @@ class List(BaseReport):
# decode column configuration
columns = [Storage(el) for el in json.loads(config.columns)]
# check column configuration
# check column configuration
# add database field map (tablename, fieldname, keyname)
# add the dataIndex (DataFrame, Ext.data.Store, Ext.grid.Panel)
map(self._check_column, columns)
# columns are persistent
self._columns = columns
# instantiate and fill the DataFrame
self._do_metric()
def _cast_type(self, column, map, xtype):
"""Cast the type of a dataframe column to the database field type
or to the grid column xtype.
The type of the column determine by the pandas might be wrong.
This append when events are merged with different user block.
This is fine in most of the case but not with computed column.
In that case the eval computation crashed.
This method avoid this problem. It also convert properly
This method avoid this problem. It also convert properly
datetime column allowing computation with them.
@type column: str
@param column: the index of the column in the DataFrame
@type map: tuple
@param map: address of the database field encoded as
(tablename, fieldname, keyname).
@type xtype: str
@param xtype: the xtype of the grid column.
Possible values are booleancolumn, datecolumn, gridcolumn
and numbercolumn.
"""
df = self.df
tablename, fieldname, keyname = map
# the dtype of column containing a mixture of type is object.
if (tablename == 'year') or (df[column].dtype != 'object'):
if (tablename == 'year') or (df[column].dtype != 'object'):
return
dbtype = self.db[tablename][fieldname].type
# the dtype for column containing string is also object
if dbtype in ('string', 'text'):
return
elif dbtype == 'boolean':
df[column] = df[column].astype('bool')
elif dbtype in ('date', 'datetime', 'time'):
df[column] = pd.to_datetime(df[column])
elif dbtype in ('double', 'integer'):
df[column] = df[column].astype('float64')
# database field containing JSON-type dictionary
# The type of the key is defined in the event model but it is
# not accessible at this stage. Instead we use the grid column xtype.
......@@ -518,51 +518,51 @@ class List(BaseReport):
elif xtype == 'booleancolumn':
df[column] = df[column].astype('bool')
elif xtype == 'datecolumn':
df[column] = pd.to_datetime(df[column])
elif xtype == 'numbercolumn':
df[column] = df[column].astype('float64')
def _check_column(self, column):
"""Check column configuration:
- Raise an exception if xtype is not defined
- Raise an exception when eval is defined but not the dataIndex
- Add the database field map
- Add the database field map
- Add the dataIndex if not defined
@type column: gluon.storage.Storage
@param column:
"""
T = current.T
xtype = column.xtype
if not xtype:
raise ReportException(T(MSG_NO_XTYPE))
if column.eval and not column.dataIndex:
raise ReportException(T(MSG_NO_DATAINDEX))
dbfield = column.dbfield
dbfield = column.dbfield
if dbfield: