Skip to content

GitLab

Commit 2ec63c24 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch '82-speedup-model' into 'master' 

Resolve "Speed up the model"

Closes #82

See merge request !83
parents a4f42353 538e4bf2
Hide whitespace changes
Inline Side-by-side
Showing with 297 additions and 571 deletions
controllers/admin.py
View file @ 2ec63c24
""" Main Controller to run the application with some privileges.
"""
from gluon import current
from auth import ADMIN, USER
@auth.requires(True, requires_login=not request.is_local)
def index():
......@@ -18,11 +21,11 @@ def index():
session.role = None
else:
return 'The user role is not defined'
return "The user role is not defined"
# no login procedure
else:
session.role = ADMIN
url = URL('plugin_dbui', 'index', args=request.args, vars=request.vars)
url = URL("plugin_dbui", "index", args=request.args, vars=request.vars)
return redirect(url)
controllers/default.py
View file @ 2ec63c24
......@@ -3,9 +3,9 @@
"""
def index():
session.role = ''
session.role = ""
url = URL('plugin_dbui', 'index', args=request.args, vars=request.vars)
url = URL("plugin_dbui", "index", args=request.args, vars=request.vars)
return redirect(url)
......
controllers/graphs.py
View file @ 2ec63c24
......@@ -4,7 +4,7 @@
"""
import base64
import matplotlib
matplotlib.use('Agg')
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from datetime import datetime
......@@ -31,19 +31,19 @@ def dashboard():
current_year = datetime.now().year
cfg = Storage()
cfg.Graph_selectorCumulative = 'True'
cfg.Graph_selectorId = ''
cfg.Graph_selectorId_authors_roles = ''
cfg.Graph_selectorId_graphs = ''
cfg.Graph_selectorId_projects = ''
cfg.Graphs_selectorId_teams = ''
cfg.Graph_selectorTime = ''
cfg.Graph_selectorYear_start = ''
cfg.Graph_selectorYear_end = ''
cfg.Graph_selectorCumulative = "True"
cfg.Graph_selectorId = ""
cfg.Graph_selectorId_authors_roles = ""
cfg.Graph_selectorId_graphs = ""
cfg.Graph_selectorId_projects = ""
cfg.Graphs_selectorId_teams = ""
cfg.Graph_selectorTime = ""
cfg.Graph_selectorYear_start = ""
cfg.Graph_selectorYear_end = ""
request.vars.update(cfg)
fields = ('cumulative','id_graphs', 'time', 'year_start','year_end')
fields = ("cumulative", "id_graphs", "time", "year_start", "year_end")
selector = Selector(virtdb.graph_selector, exclude_fields=fields)
# figure layout
......@@ -52,8 +52,8 @@ def dashboard():
# histogram of the number of publications per year
selector.cumulative = False
selector.time = T('year')
selector.year_start = ''
selector.time = T("year")
selector.year_start = ""
selector.year_end = current_year - 1
do_linechart(db.publications, selector, target=ax1)
......@@ -61,9 +61,9 @@ def dashboard():
# the cumulative sum of publications for the current year
selector.cumulative = True
selector.time = T('month')
selector.time = T("month")
selector.year_start = current_year
selector.year_end = ''
selector.year_end = ""
do_linechart(db.publications, selector, target=ax2)
......@@ -77,7 +77,9 @@ def index():
either as a linechart or as a stacked histograms.
"""
fields = ('author', 'cumulative','id_graphs', 'time', 'year_start','year_end')
fields = \
("author", "cumulative", "id_graphs", "time", "year_start", "year_end")
selector = Selector(virtdb.graph_selector, exclude_fields=fields)
# graph configuration
......
controllers/harvest.py
View file @ 2ec63c24
......@@ -19,8 +19,7 @@ from invenio_tools import (load_record,
RecordConf,
RecordThesis,
REG_INT)
from plugin_dbui import (get_id,
inline_alert,
from plugin_dbui import (inline_alert,
Selector,
to_formPanel,
UNDEF_ID)
......@@ -30,6 +29,7 @@ MSG_NO_AFFILIATION = "Affiliation keys are not defined !!!"
MSG_NO_HARVESTER = "No harvesters for your selection !!!"
MSG_NO_RECORD = "Sorry, the record does not exist."
def free_run():
"""Run a free harvester.
All harvester parameters are defined via the selector.
......@@ -39,36 +39,39 @@ def free_run():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
table = virtdb.free_harvester_selector
fields = ('collections',
'controller',
'host',
'id_projects',
'id_teams',
'id_categories',
'ratio')
fields = ("collections",
"controller",
"host",
"id_projects",
"id_teams",
"id_categories",
"ratio")
try:
selector = Selector(table,
exclude_fields=('mode', 'year_start', 'year_end'))
selector = Selector(
table,
exclude_fields=("mode", "year_start", "year_end"))
for el in fields:
if not selector[el]:
msg = T('All fields of the form have to be defined !!!')
msg = T("All fields of the form have to be defined !!!")
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return inline_alert(T('Error'), msg)
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
msg += T("The field '%s' is missing ...") % T(table[el].label)
return inline_alert(T("Error"), msg)
tool = build_harvester_tool(
db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.process_url(selector.host, selector.collections)
......@@ -76,14 +79,14 @@ def free_run():
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
response.view = 'harvest/layout.html'
response.view = "harvest/layout.html"
report = tool.report()
report['selector'] = selector
report["selector"] = selector
return report
......@@ -98,12 +101,13 @@ def edit_insert():
if db(db.affiliation_keys.id > 0).count() == 0:
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
fields = ('controller',
'host',
'id_projects',
'id_teams',
'id_categories',
'record_id')
fields = (
"controller",
"host",
"id_projects",
"id_teams",
"id_categories",
"record_id")
table = virtdb.edit_insert_selector
......@@ -118,22 +122,22 @@ def edit_insert():
msg = T("The <i>record id</i> is not well formed.")
msg += "<br>"
msg += T("Use only digit character, no comma, no dot...")
return inline_alert(T('Error'), msg)
return inline_alert(T("Error"), msg)
selector = Selector(table)
for el in fields:
if not selector[el]:
msg = T('All fields of the form have to be defined !!!')
msg = T("All fields of the form have to be defined !!!")
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return inline_alert(T('Error'), msg)
msg += T("The field '%s' is missing ...") % T(table[el].label)
return inline_alert(T("Error"), msg)
# record
record = load_record(selector.host, selector.record_id)
if record is None:
return inline_alert(T('Error'), T(MSG_NO_RECORD))
return inline_alert(T("Error"), T(MSG_NO_RECORD))
# form configuration
cfg = to_formPanel(db.publications)
......@@ -146,17 +150,19 @@ def edit_insert():
check.recover_oai(record, selector.host)
# title, preprint, URL, report number
values['PublicationsTitle'] = record.title()
values['PublicationsPreprint'] = record.preprint_number()
values['PublicationsPublication_url'] = record.paper_url()
values['PublicationsReport_numbers'] = record.report_number()
values["PublicationsTitle"] = record.title()
values["PublicationsPreprint"] = record.preprint_number()
values["PublicationsPublication_url"] = record.paper_url()
values["PublicationsReport_numbers"] = record.report_number()
# authors
try:
check.authors(record)
check.format_authors(record, format_author_fr)
check.my_affiliation(record, selector.id_projects, selector.id_teams)
check.my_affiliation(
record, selector.id_projects, selector.id_teams)
check.get_my_authors(record, cmpFct=family_name_fr)
except CheckException:
......@@ -166,9 +172,9 @@ def edit_insert():
if isinstance(fauthor, list):
fauthor = u", ".join(fauthor)
values['PublicationsFirst_author'] = fauthor
values['PublicationsAuthors'] = record.authors()
values['PublicationsAuthors_institute'] = record.my_authors
values["PublicationsFirst_author"] = fauthor
values["PublicationsAuthors"] = record.authors()
values["PublicationsAuthors_institute"] = record.my_authors
# collaboration
recId = UNDEF_ID
......@@ -179,12 +185,12 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_collaborations'] = int(recId)
values["PublicationsId_collaborations"] = int(recId)
# teams, project, categories
values['PublicationsId_categories'] = int(selector.id_categories)
values['PublicationsId_projects'] = int(selector.id_projects)
values['PublicationsId_teams'] = int(selector.id_teams)
values["PublicationsId_categories"] = int(selector.id_categories)
values["PublicationsId_projects"] = int(selector.id_projects)
values["PublicationsId_teams"] = int(selector.id_teams)
# origin
# Note:
......@@ -194,10 +200,10 @@ def edit_insert():
if not oai_url:
oai_url = OAI_URL % (selector.host, selector.record_id)
values['PublicationsOrigin'] = oai_url
values["PublicationsOrigin"] = oai_url
# publishers
if selector.controller in ('articles', 'proceedings'):
if selector.controller in ("articles", "proceedings"):
check.clean_erratum(record)
check.paper_reference(record)
......@@ -211,12 +217,12 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_publishers'] = int(recId)
values['PublicationsVolume'] = record.paper_volume()
values['PublicationsPages'] = record.paper_pages()
values["PublicationsId_publishers"] = int(recId)
values["PublicationsVolume"] = record.paper_volume()
values["PublicationsPages"] = record.paper_pages()
# conference
if selector.controller in ('proceedings', 'talks'):
if selector.controller in ("proceedings", "talks"):
try:
check.country(record)
......@@ -226,10 +232,17 @@ def edit_insert():
pass
if isinstance(record, RecordConf):
values['PublicationsConference_title'] = record.conference_title()
values['PublicationsConference_url'] = record.conference_url()
values['PublicationsConference_dates'] = record.conference_dates()
values['PublicationsConference_town'] = record.conference_town()
values["PublicationsConference_title"] = \
record.conference_title()
values["PublicationsConference_url"] = \
record.conference_url()
values["PublicationsConference_dates"] = \
record.conference_dates()
values["PublicationsConference_town"] = \
record.conference_town()
recId = UNDEF_ID
try:
......@@ -239,16 +252,20 @@ def edit_insert():
except ToolException:
pass
values['PublicationsId_countries'] = int(recId)
values['PublicationsConference_speaker'] = record.first_author()
values["PublicationsId_countries"] = int(recId)
values["PublicationsConference_speaker"] = \
record.first_author()
# thesis
if selector.controller == 'theses':
if selector.controller == "theses":
if isinstance(record, RecordThesis):
values['PublicationsUniversities'] = record.these_universities()
values['PublicationsDirectors'] = record.these_directors()
values['PublicationsDefense'] = record.these_defense()
values["PublicationsUniversities"] = \
record.these_universities()
values["PublicationsDirectors"] = record.these_directors()
values["PublicationsDefense"] = record.these_defense()
# submitted date and year
try:
......@@ -257,21 +274,21 @@ def edit_insert():
except CheckException:
pass
values['PublicationsSubmitted'] = ', '.join(record.submitted())
values["PublicationsSubmitted"] = ", ".join(record.submitted())
if record.is_published():
year = record.paper_year()
else:
year = record.year()
values['PublicationsYear'] = year
values["PublicationsYear"] = year
except Exception:
# log the exception in the web2py ticker system
ticket = RestrictedError(layer='harvester.py',
code='edit_insert',
output='',
ticket = RestrictedError(layer="harvester.py",
code="edit_insert",
output="",
environment=current.globalenv)
ticket.log(request)
......@@ -289,19 +306,21 @@ def insert_marcxml():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
try:
selector = Selector(virtdb.marc12_selector, exclude_fields=('mode'))
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
selector = Selector(virtdb.marc12_selector, exclude_fields=("mode"))
tool = build_harvester_tool(
db,
selector.id_teams,
selector.id_projects,
selector.controller,
selector.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.process_xml(selector.xml)
......@@ -309,14 +328,14 @@ def insert_marcxml():
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
response.view = 'harvest/layout.html'
response.view = "harvest/layout.html"
report = tool.report()
report['selector'] = selector
report["selector"] = selector
return report
......@@ -337,8 +356,9 @@ def run():
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
try:
selector = Selector(virtdb.harvester_selector,
exclude_fields=('mode', 'year_start', 'year_end'))
selector = Selector(
virtdb.harvester_selector,
exclude_fields=("mode", "year_start", "year_end"))
# Get hosts and collections
rows = selector.select(db.harvesters)
......@@ -349,17 +369,19 @@ def run():
logs = []
for row in rows:
tool = build_harvester_tool(db,
selector.id_teams,
selector.id_projects,
selector.controller,
row.harvesters.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
tool = build_harvester_tool(
db,
selector.id_teams,
selector.id_projects,
selector.controller,
row.harvesters.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.process_url(row.harvesters.host, row.harvesters.collections)
......@@ -370,13 +392,13 @@ def run():
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
# delegate rendering to the report view
response.view = 'harvest/layout.%s' % request.extension
response.view = "harvest/layout.%s" % request.extension
return dict(collection_logs=collection_logs,
controller=selector.controller,
logs=logs,
......@@ -394,35 +416,35 @@ def run_all():
logs = []
try:
selector = Selector(virtdb.run_all_harvesters_selector,
exclude_fields=('mode', 'year_start', 'year_end'))
selector = Selector(
virtdb.run_all_harvesters_selector,
exclude_fields=("mode", "year_start", "year_end"))
query = None
for fieldname in ('id_teams', 'id_projects'):
for fieldname in ("id_teams", "id_projects"):
if selector[fieldname]:
q = db.harvesters[fieldname] == selector[fieldname]
if query:
query = (query) & (q)
else:
query = q
query = (q if query is None else (query) & (q))
harvesters = db(query).select(db.harvesters.ALL)
if not len(harvesters):
return inline_alert(T('Error'), T(MSG_NO_HARVESTER))
return inline_alert(T("Error"), T(MSG_NO_HARVESTER))
for harvester in harvesters:
tool = build_harvester_tool(db,
harvester.id_teams,
harvester.id_projects,
harvester.controller,
harvester.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
tool = build_harvester_tool(
db,
harvester.id_teams,
harvester.id_projects,
harvester.controller,
harvester.id_categories,
year_start=selector.year_start,
year_end=selector.year_end,
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return inline_alert(T('Error'), T('Select an harvester.'))
return inline_alert(T("Error"), T("Select an harvester."))
tool.process_url(harvester.host, harvester.collections)
......@@ -433,9 +455,9 @@ def run_all():
return T(str(e))
except BaseException as e:
msg = '<br><br><hr/>'
msg = "<br><br><hr/>"
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
msg += "<hr/>"
return msg
# tune selector parameters used in the report title
......@@ -443,8 +465,8 @@ def run_all():
selector.id_projects = None
# delegate rendering to the report view
response.view = 'harvest/layout.%s' % request.extension
response.view = "harvest/layout.%s" % request.extension
return dict(collection_logs=collection_logs,
controller='all harvesters',
controller="all harvesters",
logs=logs,
selector=selector)
controllers/lists.py
View file @ 2ec63c24
""" Controllers handling list
"""
import re
from gluon.dal import Field
from plugin_dbui import Selector
from reporting_tools import (get_converter,
......@@ -21,11 +18,9 @@ def index():
# get the user constraint.
# the selector fields year_start, year_end, id_lists are excluded
# from the selector query.
selector = Selector(virtdb.list_selector,
exclude_fields=('author',
'year_start',