Commit 78eee94f authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production'

Release 0.9.6.5

See merge request !89
parents 0fe214bd 9e378e38
......@@ -13,6 +13,7 @@ databases
databases*/
errors/
private/
prof/
sessions/
uploads/
models/plugin_dbui.py
......
......@@ -2,6 +2,18 @@
HEAD
0.9.6.5 (Jun 2017)
- Require plugin_dbui 0.9.8.2 or higher.
- Factorize the models in classes.
- Run pylint and replace single quote by double one.
- Activate the lazy Translation.
- Speedup the graph by delegating the dirty work to DataFrame.
- Dashboard can be plot as function of year of submission or publication.
- Restore the action insert MarcXML.
- Authors and their affiliations are store in a DataFrame structure.
- Use the module requests instead of urllib.
- Modify the logic for publications marked OK. Now they can be unlocked.
0.9.6.4 (Apr 2017)
- Migrate to plugin_dbui 0.9.8.1
- This version is not backward compatible due to changes in plugin_dbui.
......
0.9.6.4
\ No newline at end of file
0.9.6.5
\ No newline at end of file
""" Main Controller to run the application with some privileges.
"""
from gluon import current
from auth import ADMIN, USER
@auth.requires(True, requires_login=not request.is_local)
def index():
......@@ -18,11 +21,11 @@ def index():
session.role = None
else:
return 'The user role is not defined'
return "The user role is not defined"
# no login procedure
else:
session.role = ADMIN
url = URL('plugin_dbui', 'index', args=request.args, vars=request.vars)
url = URL("plugin_dbui", "index", args=request.args, vars=request.vars)
return redirect(url)
......@@ -3,9 +3,9 @@
"""
def index():
session.role = ''
session.role = ""
url = URL('plugin_dbui', 'index', args=request.args, vars=request.vars)
url = URL("plugin_dbui", "index", args=request.args, vars=request.vars)
return redirect(url)
......
......@@ -4,22 +4,17 @@
"""
import base64
import matplotlib
matplotlib.use('Agg')
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from datetime import datetime
from gluon import current
from gluon.storage import Storage
from graph_tools import (do_labels,
do_linechart,
do_stackedchart,
do_title,
savefig)
from pandas import DataFrame
from plugin_dbui import Selector
from reporting_tools import repr_team_project
TITLE_Y = "Number of publications"
from graph_tools import (FROM_TO,
LABELY_YEAR,
linechart,
savefig,
stackchart)
from model_selector import YEAR_SUBMIT
def dashboard():
......@@ -28,71 +23,70 @@ def dashboard():
the current year.
"""
current_year = datetime.now().year
cfg = Storage()
cfg.Graph_selectorCumulative = 'True'
cfg.Graph_selectorId = ''
cfg.Graph_selectorId_authors_roles = ''
cfg.Graph_selectorId_graphs = ''
cfg.Graph_selectorId_projects = ''
cfg.Graphs_selectorId_teams = ''
cfg.Graph_selectorTime = ''
cfg.Graph_selectorYear_start = ''
cfg.Graph_selectorYear_end = ''
request.vars.update(cfg)
fields = ('cumulative','id_graphs', 'time', 'year_start','year_end')
selector = Selector(virtdb.graph_selector, exclude_fields=fields)
current_year = request.now.year
dashboard_start = current.app.dashboard_start_year
selector = Storage()
selector.Graph_selectorCumulative = "true"
selector.Graph_selectorId = ""
selector.Graph_selectorId_authors_roles = ""
selector.Graph_selectorId_graphs = ""
selector.Graph_selectorId_projects = ""
selector.Graphs_selectorId_teams = ""
selector.Graph_selectorTime = ""
selector.Graph_selectorYear_start = ""
selector.Graph_selectorYear_end = ""
# figure layout
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, sharey=True)
fig.subplots_adjust(wspace=0.01)
# histogram of the number of publications per year
selector.cumulative = False
selector.time = T('year')
selector.year_start = ''
selector.year_end = current_year - 1
# the number of publications per year except for the current year
# the starting value is defined in the preferences of the application
min_year = ("" if dashboard_start is None else str(dashboard_start))
max_year = str(current_year - 1)
do_linechart(db.publications, selector, target=ax1)
do_labels(ax1, "", T(TITLE_Y))
selector.Graph_selectorCumulative = "false"
selector.Graph_selectorTime = T(YEAR_SUBMIT)
selector.Graph_selectorYear_start = min_year
selector.Graph_selectorYear_end = max_year
# the cumulative sum of publications for the current year
selector.cumulative = True
selector.time = T('month')
selector.year_start = current_year
selector.year_end = ''
title = (T(FROM_TO) % (min_year, max_year)).decode("utf-8")
ylabel = T(LABELY_YEAR).decode("utf-8")
do_linechart(db.publications, selector, target=ax2)
linechart(db,
selector,
target=ax1,
xlabel="",
ylabel=ylabel,
title=title)
# the cumulative sum of publications for the current year on a month
year = str(current_year)
selector.Graph_selectorCumulative = "true"
selector.Graph_selectorTime = T(YEAR_SUBMIT)
selector.Graph_selectorYear_start = year
selector.Graph_selectorYear_end = ""
title = (T("In %s") % year).decode("utf-8")
linechart(db, selector, target=ax2, title=title)
# delegate the rendering to the view
response.view = "graphs/index.html"
return dict(data=savefig(fig, "svg"))
def index():
def publications_versus_time():
"""Generate graph showing the number of publication per month / year
either as a linechart or as a stacked histograms.
either as a line chart or as a stacked histograms.
"""
fields = ('author', 'cumulative','id_graphs', 'time', 'year_start','year_end')
selector = Selector(virtdb.graph_selector, exclude_fields=fields)
# graph configuration
graph = db.graphs[selector.id_graphs]
# stacked chart
if graph.stack_axis:
ax = do_stackedchart(db.publications, selector, graph)
# line char
else:
ax = do_linechart(db.publications, selector)
# user criteria and graph configuration
selector = request.vars
graph = db.graphs[selector.Graph_selectorId_graphs]
axis = graph.stack_axis
do_labels(ax, "", T(TITLE_Y))
do_title(ax, db, selector)
# instantiate the graph
ax = (stackchart(db, selector) if axis else linechart(db, selector))
# delegate the rendering to the view
extension = request.extension
......@@ -103,4 +97,5 @@ def index():
if fmt in ("pdf", "png"):
data = base64.b64encode(data)
response.view = "graphs/index.%s" % extension
return dict(data=data)
......@@ -10,8 +10,6 @@ from harvest_tools import (build_harvester_tool,
CheckAndFix,
CheckException,
DRY_RUN,
format_author_fr,
family_name_fr,
search_synonym,
ToolException)
from invenio_tools import (load_record,
......@@ -19,8 +17,7 @@ from invenio_tools import (load_record,
RecordConf,
RecordThesis,
REG_INT)
from plugin_dbui import (get_id,
inline_alert,
from plugin_dbui import (inline_alert,
Selector,
to_formPanel,
UNDEF_ID)
......@@ -30,6 +27,7 @@ MSG_NO_AFFILIATION = "Affiliation keys are not defined !!!"
MSG_NO_HARVESTER = "No harvesters for your selection !!!"
MSG_NO_RECORD = "Sorry, the record does not exist."
def free_run():
"""Run a free harvester.
All harvester parameters are defined via the selector.
......@@ -39,36 +37,39 @@ def free_run():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
table = virtdb.free_harvester_selector
fields = ('collections',
'controller',
'host',
'id_projects',
'id_teams',
'id_categories',
'ratio')
fields = ("collections",
"controller",
"host",
"id_projects",
"id_teams",
"id_categories",
"ratio")
try:
selector = Selector(table,
exclude_fields=('mode', 'year_start', 'year_end'))
selector = Selector(
table,
exclude_fields=("mode", "year_start", "year_end"))
for el in fields:
if not selector[el]:
msg = T('All fields of the form have to be defined !!!')
msg = T("All fields of the form have to be defined !!!")
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return inline_alert(T('Error'), msg)
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
msg += T("The field '%s' is missing ...") % T(table[el].label)
return inline_alert(T("Error"), msg)
tool = build_harvester_tool(
db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.process_url(selector.host, selector.collections)
......@@ -76,14 +77,14 @@ def free_run():
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
response.view = 'harvest/layout.html'
response.view = "harvest/layout.html"
report = tool.report()
report['selector'] = selector
report["selector"] = selector
return report
......@@ -98,12 +99,13 @@ def edit_insert():
if db(db.affiliation_keys.id > 0).count() == 0:
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
fields = ('controller',
'host',
'id_projects',
'id_teams',
'id_categories',
'record_id')
fields = (
"controller",
"host",
"id_projects",
"id_teams",
"id_categories",
"record_id")
table = virtdb.edit_insert_selector
......@@ -118,22 +120,22 @@ def edit_insert():
msg = T("The <i>record id</i> is not well formed.")
msg += "<br>"
msg += T("Use only digit character, no comma, no dot...")
return inline_alert(T('Error'), msg)
return inline_alert(T("Error"), msg)
selector = Selector(table)
for el in fields:
if not selector[el]:
msg = T('All fields of the form have to be defined !!!')
msg = T("All fields of the form have to be defined !!!")
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return inline_alert(T('Error'), msg)
msg += T("The field '%s' is missing ...") % T(table[el].label)
return inline_alert(T("Error"), msg)
# record
record = load_record(selector.host, selector.record_id)
if record is None:
return inline_alert(T('Error'), T(MSG_NO_RECORD))
return inline_alert(T("Error"), T(MSG_NO_RECORD))
# form configuration
cfg = to_formPanel(db.publications)
......@@ -146,18 +148,20 @@ def edit_insert():
check.recover_oai(record, selector.host)
# title, preprint, URL, report number
values['PublicationsTitle'] = record.title()
values['PublicationsPreprint'] = record.preprint_number()
values['PublicationsPublication_url'] = record.paper_url()
values['PublicationsReport_numbers'] = record.report_number()
values["PublicationsTitle"] = record.title()
values["PublicationsPreprint"] = record.preprint_number()
values["PublicationsPublication_url"] = record.paper_url()
values["PublicationsReport_numbers"] = record.report_number()
# authors
try:
check.authors(record)
check.format_authors(record, format_author_fr)
check.format_authors(record, fmt="F. Last")
check.my_affiliation(
record, selector.id_projects, selector.id_teams)
check.my_affiliation(record, selector.id_projects, selector.id_teams)
check.get_my_authors(record, cmpFct=family_name_fr)
check.get_my_authors(record, sort=True)
except CheckException:
pass
......@@ -166,9 +170,9 @@ def edit_insert():
if isinstance(fauthor, list):
fauthor = u", ".join(fauthor)
values['PublicationsFirst_author'] = fauthor
values['PublicationsAuthors'] = record.authors()
values['PublicationsAuthors_institute'] = record.my_authors
values["PublicationsFirst_author"] = fauthor
values["PublicationsAuthors"] = record.authors()
values["PublicationsAuthors_institute"] = record.my_authors
# collaboration
recId = UNDEF_ID
......@@ -179,12 +183,12 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_collaborations'] = int(recId)
values["PublicationsId_collaborations"] = int(recId)
# teams, project, categories
values['PublicationsId_categories'] = int(selector.id_categories)
values['PublicationsId_projects'] = int(selector.id_projects)
values['PublicationsId_teams'] = int(selector.id_teams)
values["PublicationsId_categories"] = int(selector.id_categories)
values["PublicationsId_projects"] = int(selector.id_projects)
values["PublicationsId_teams"] = int(selector.id_teams)
# origin
# Note:
......@@ -194,10 +198,10 @@ def edit_insert():
if not oai_url:
oai_url = OAI_URL % (selector.host, selector.record_id)
values['PublicationsOrigin'] = oai_url
values["PublicationsOrigin"] = oai_url
# publishers
if selector.controller in ('articles', 'proceedings'):
if selector.controller in ("articles", "proceedings"):
check.clean_erratum(record)
check.paper_reference(record)
......@@ -211,12 +215,12 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_publishers'] = int(recId)
values['PublicationsVolume'] = record.paper_volume()
values['PublicationsPages'] = record.paper_pages()
values["PublicationsId_publishers"] = int(recId)
values["PublicationsVolume"] = record.paper_volume()
values["PublicationsPages"] = record.paper_pages()
# conference
if selector.controller in ('proceedings', 'talks'):
if selector.controller in ("proceedings", "talks"):
try:
check.country(record)
......@@ -226,10 +230,17 @@ def edit_insert():
pass
if isinstance(record, RecordConf):
values['PublicationsConference_title'] = record.conference_title()
values['PublicationsConference_url'] = record.conference_url()
values['PublicationsConference_dates'] = record.conference_dates()
values['PublicationsConference_town'] = record.conference_town()
values["PublicationsConference_title"] = \
record.conference_title()
values["PublicationsConference_url"] = \
record.conference_url()
values["PublicationsConference_dates"] = \
record.conference_dates()
values["PublicationsConference_town"] = \
record.conference_town()
recId = UNDEF_ID
try:
......@@ -239,16 +250,20 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_countries'] = int(recId)
values['PublicationsConference_speaker'] = record.first_author()
values["PublicationsId_countries"] = int(recId)
values["PublicationsConference_speaker"] = \
record.first_author()
# thesis
if selector.controller == 'theses':
if selector.controller == "theses":
if isinstance(record, RecordThesis):
values['PublicationsUniversities'] = record.these_universities()
values['PublicationsDirectors'] = record.these_directors()
values['PublicationsDefense'] = record.these_defense()
values["PublicationsUniversities"] = \
record.these_universities()
values["PublicationsDirectors"] = record.these_directors()
values["PublicationsDefense"] = record.these_defense()
# submitted date and year
try:
......@@ -257,21 +272,21 @@ def edit_insert():
except CheckException:
pass
values['PublicationsSubmitted'] = ', '.join(record.submitted())
values["PublicationsSubmitted"] = ", ".join(record.submitted())
if record.is_published():
year = record.paper_year()
else:
year = record.year()
values['PublicationsYear'] = year
values["PublicationsYear"] = year
except Exception:
# log the exception in the web2py ticker system
ticket = RestrictedError(layer='harvester.py',
code='edit_insert',
output='',
ticket = RestrictedError(layer="harvester.py",
code="edit_insert",
output="",
environment=current.globalenv)
ticket.log(request)
......@@ -289,34 +304,37 @@ def insert_marcxml():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
try:
selector = Selector(virtdb.marc12_selector, exclude_fields=('mode'))
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
selector = Selector(virtdb.marc12_selector, exclude_fields=("mode"))
tool = build_harvester_tool(
db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.harvester.host = selector.host
tool.process_xml(selector.xml)
except ToolException as e:
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
response.view = 'harvest/layout.html'
response.view = "harvest/layout.html"
report = tool.report()
report['selector'] = selector
report["selector"] = selector
return report
......@@ -337,8 +355,9 @@ def run():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
try:
selector = Selector(virtdb.harvester_selector,
exclude_fields=('mode', 'year_start', 'year_end'))
selector = Selector(
virtdb.harvester_selector,
exclude_fields=("mode", "year_start", "year_end"))
# Get hosts and collections
rows = selector.select(db.harvesters)
......@@ -349,17 +368,19 @@ def run():
logs = []
for row in rows:
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
row.harvesters.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),