Commit 96ea84b9 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production'

Release 0.9.3

* Add mainly the wizard to create harvester
* Require `plugin_dbui` 0.7.3

See merge request !49
parents 6da2f1dd 8d1a9ae4
......@@ -23,6 +23,6 @@ static/plugin_ace
static/plugin_dbui
static/plugin_extjs
static/plugin_mathjax
static/track-publication*.js
static/track_publications*.js
views/plugin_dbui
controllers/plugin_dbui.py
......@@ -36,6 +36,7 @@ from subprocess import call
APP = os.path.basename(os.getcwd())
API = 'api'
CHANGELOG = 'static/CHANGELOG'
DBUISRC = 'static/plugin_dbui/src'
DOCS = 'static/docs'
DOCSRC = 'docs'
EXTJSSRC = 'static/plugin_extjs/src'
......@@ -202,22 +203,20 @@ def compile_js():
print 'Remove old javascript library', item
# debug version of the javascript library
cwd = os.getcwd()
cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
"compile", "-class", opj(cwd, JSLIBSRC),
"exclude", "--namespace", "Ext",
"and", "concat", opj(cwd, JSLIBDEBUG)]
cmd = ["sencha", "-sdk", EXTJSSRC,
"compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
"exclude", "--namespace", "Ext,App",
"and", "concat", JSLIBDEBUG]
call(cmd)
print 'Debug version of the javascript library', JSLIBDEBUG, 'is ready'
# Minified version of the javascript library
cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
"compile", "-class", opj(cwd, JSLIBSRC),
"exclude", "--namespace", "Ext",
"and", "concat", "--yui", opj(cwd, JSLIBMIN)]
cmd = ["sencha", "-sdk", EXTJSSRC,
"compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
"exclude", "--namespace", "Ext,App",
"and", "concat", "--yui", JSLIBMIN]
call(cmd)
......@@ -419,7 +418,9 @@ if __name__ == '__main__':
APS.add_argument("-C", "--commit-changelog",
action="store_true",
help="commit CHANGELOG.")
help="commit CHANGELOG. "
"To be used with --write-release. "
"Recommend to use --start-release.")
APS.add_argument("-j", "--jsduck",
action="store_true",
......@@ -435,7 +436,11 @@ if __name__ == '__main__':
APS.add_argument("-s", "--start-release",
action="store_true",
help="start the new release.")
help="start the new release cycle. "
"Set the release number in the changelog and "
"commit changes in the master branch. "
"Equivalent to --write-release followed by "
"--commit-changelog.")
APS.add_argument("-u", "--user-doc",
action="store_true",
......@@ -451,7 +456,9 @@ if __name__ == '__main__':
APS.add_argument("-w", "--write-release",
action="store_true",
help="write the release number in CHANGELOG.")
help="write the release number in CHANGELOG. "
"To be used with --commit-changelog. "
"Recommend to use --start-release.")
ARGS = APS.parse_args()
......
......@@ -20,7 +20,7 @@ from invenio_tools import (load_record,
RecordThesis,
REG_INT)
from plugin_dbui import (get_id,
INLINE_ALERT,
inline_alert,
Selector,
to_formPanel,
UNDEF_ID)
......@@ -36,7 +36,7 @@ def free_run():
"""
if not current.app.inspirehep_institute_id:
return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
table = virtdb.free_harvester_selector
fields = ('collections',
......@@ -56,7 +56,7 @@ def free_run():
msg = T('All fields of the form have to be defined !!!')
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return INLINE_ALERT % (T('Error'), msg)
return inline_alert(T('Error'), msg)
tool = build_harvester_tool(db,
selector.id_teams,
......@@ -68,7 +68,7 @@ def free_run():
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
return inline_alert(T('Error'), T('Select an harvester.'))
tool.process_url(selector.host, selector.collections)
......@@ -96,7 +96,7 @@ def edit_insert():
"""
if not current.app.inspirehep_institute_id:
return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
fields = ('controller',
'host',
......@@ -118,7 +118,7 @@ def edit_insert():
msg = T("The <i>record id</i> is not well formed.")
msg += "<br>"
msg += T("Use only digit character, no comma, no dot...")
return INLINE_ALERT % (T('Error'), msg)
return inline_alert(T('Error'), msg)
selector = Selector(table)
......@@ -127,17 +127,13 @@ def edit_insert():
msg = T('All fields of the form have to be defined !!!')
msg += "<br>"
msg += T('The field "%s" is missing ...') % T(table[el].label)
return INLINE_ALERT % (T('Error'), msg)
return inline_alert(T('Error'), msg)
# record
record = load_record(selector.host, selector.record_id)
if record is None:
# NOTE
# Bug in plugin dbui 0.7.1 in INLINE_ALERT
msg = '<script>Ext.Msg.alert("%s", "%s");</script>'
msg = msg % (T('Error'), T(MSG_NO_RECORD))
return msg
return inline_alert(T('Error'), T(MSG_NO_RECORD))
# form configuration
cfg = to_formPanel(db.publications)
......@@ -283,7 +279,7 @@ def insert_marcxml():
"""
if not current.app.inspirehep_institute_id:
return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
try:
selector = Selector(virtdb.marc12_selector, exclude_fields=('mode'))
......@@ -298,7 +294,7 @@ def insert_marcxml():
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
return inline_alert(T('Error'), T('Select an harvester.'))
tool.process_xml(selector.xml)
......@@ -331,7 +327,7 @@ def run():
"""
if not current.app.inspirehep_institute_id:
return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
try:
selector = Selector(virtdb.harvester_selector,
......@@ -356,7 +352,7 @@ def run():
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
return inline_alert(T('Error'), T('Select an harvester.'))
tool.process_url(row.harvesters.host, row.harvesters.collections)
......@@ -385,7 +381,7 @@ def run_all():
"""
if not current.app.inspirehep_institute_id:
return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
collection_logs = []
logs = []
......@@ -405,7 +401,7 @@ def run_all():
harvesters = db(query).select(db.harvesters.ALL)
if not len(harvesters):
return INLINE_ALERT % (T('Error'), T(MSG_NO_HARVESTER))
return inline_alert(T('Error'), T(MSG_NO_HARVESTER))
for harvester in harvesters:
......@@ -419,7 +415,7 @@ def run_all():
dry_run=(selector.mode == MODE_DRY_RUN),
debug=False)
if not tool:
return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
return inline_alert(T('Error'), T('Select an harvester.'))
tool.process_url(harvester.host, harvester.collections)
......
......@@ -5,12 +5,10 @@ import json
import re
from gluon.dal import smart_query
from plugin_dbui import Selector
from plugin_dbui import inline_alert, Selector
from reporting_tools import get_converter, repr_team_project
INLINE_ALERT = "<script>Ext.Msg.alert('%s', '%s');</script>"
MSG_NO_METRIC = T("Please select a metric....")
MSG_NO_TABLE = T("Invalid database table '%s'")
MSG_NO_METRIC = "Please select a metric...."
def index():
......@@ -27,7 +25,7 @@ def index():
# protection
if not selector.id_metrics:
return INLINE_ALERT % (T("Error"), MSG_NO_METRIC)
return inline_alert(T("Error"), T(MSG_NO_METRIC))
# retrieve metric data
metric = db.metrics[selector.id_metrics]
......
......@@ -8,18 +8,18 @@ import re
from check_tools import check_publication
from gluon.storage import Storage
from harvest_tools import DRY_RUN
from plugin_dbui import (is_foreign_field,
from plugin_dbui import (CALLBACK_ERRORS,
inline_alert,
is_foreign_field,
get_foreign_field,
get_id,
Selector,
to_fields)
MODE_DRY_RUN = T(DRY_RUN)
INLINE_ALERT = "<script>Ext.Msg.alert('%s', '%s');</script>"
MSG_NO_AUTHORS = "<br><br>Removing affiliation failed.<br>"\
"Use INSPIRES instead with the tool 'insert MARCXML'"
"Use INSPIRES instead with the tool 'insert MARCXML'"
def check_validate():
......@@ -28,7 +28,7 @@ def check_validate():
"""
counters = {}
logs = []
id_ok = db(db.status.code=='OK').select().first().id
id_ok = db(db.status.code == 'OK').select().first().id
# get user requirement
selector = Selector(virtdb.check_selector, exclude_fields=('mode'))
......@@ -89,7 +89,7 @@ def compare_publications():
data, idrow = [], []
if 'id1' not in request.vars or 'id2' not in request.vars:
return INLINE_ALERT % (T('Error'), T('Specify id1 and id2 in the URL'))
return inline_alert(T('Error'), T('Specify id1 and id2 in the URL'))
row1 = db.publications[request.vars.id1]
row2 = db.publications[request.vars.id2]
......@@ -188,4 +188,116 @@ def extract_authors():
if not case_1:
return MSG_NO_AUTHORS
return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
\ No newline at end of file
return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
def harvester():
"""Process the data send by the wizard harvester and fill the
harvesters table. The data block contains the keys:
* automaton (str)
* collaboration (str)
* collection (str)
* people (list)
* producer (str) either 'collaboration' or 'people'
* project (int)
* store (str) either cds.cern.ch or inspirehep.net
The logic to interpret the data depends on the store.
"""
# shortcuts
vars = request.vars
automaton = vars.automaton
organisation = db.organisation
project = vars.project
store = vars.store
# protection
if store not in ("cds.cern.ch", "inspirehep.net"):
raise HTTP(500, "Invalid store !")
values = Storage(controller=automaton,
host=store,
id_projects=project)
# the team
rec_id = get_id(organisation, id_projects=project)
if rec_id is None:
raise HTTP(500, "Project is unknown !")
values.id_teams = organisation[rec_id].id_teams
# the collection
if store == "cds.cern.ch":
values.collections = vars.collection
elif store == "inspirehep.net":
collection = []
collaboration = vars.collaboration
if collaboration:
collection.append("cn %s" % collaboration)
else:
collection.append("(a %s)" % " or ".join(vars.people))
if automaton == "articles":
collection.append("tp p and not tc c")
elif automaton == "proceedings":
collection.append("tp p and tc c")
values.collections = "find %s" % " and ".join(collection)
# the publication category
if automaton == "articles":
id_category = get_id(db.categories, code="ACL")
elif automaton in ("notes", "reports"):
id_category = get_id(db.categories, code="AP")
elif automaton == "preprints":
id_category = get_id(db.categories, code="PRE")
elif automaton == "proceedings":
id_category = get_id(db.categories, code="ACTI")
elif automaton == "talks":
id_category = get_id(db.categories, code="COM")
elif automaton == "theses":
id_category = get_id(db.categories, code="PHD")
if id_category is None:
raise HTTP(500, "Category is unknown !")
values.id_categories = id_category
# insert new values in the database
msg = None
try:
rec_id = db.harvesters.insert(**values)
if rec_id:
return
# operation can be reject by callback table._before_insert
else:
msg = "Fail to insert the new harvester in the database."
if CALLBACK_ERRORS in db.harvesters:
msg = db.harvesters._callback_errors
# reduce the error message
if isinstance(msg, list):
msg = "%s %s" % (msg[0], msg[-1])
# operation is rejected by the database
except Exception as dbe:
raise HTTP(500, dbe.message)
# operation is rejected by the callback
# NOTE in the else branch to avoid recursive exception generation
if msg is not None:
raise HTTP(500, msg)
return
callbacks.INHIBIT_HARVESTER
============================
.. currentmodule:: callbacks
.. autofunction:: INHIBIT_HARVESTER
\ No newline at end of file
callbacks.INHIBIT_HARVESTER_ON_CATEGORY
=======================================
.. currentmodule:: callbacks
.. autofunction:: INHIBIT_HARVESTER_ON_CATEGORY
\ No newline at end of file
......@@ -23,7 +23,7 @@ the-database-abstraction-layer#callbacks-on-record-insert--delete-and-update>`_.
~INHIBIT_CASCADE_DELETE
~INHIBIT_DUPLICATE_PUBLICATION
~INHIBIT_HARVESTER_ON_CATEGORY
~INHIBIT_HARVESTER
~INHIBIT_PUBLICATION_DELETE_ON_OK
~INHIBIT_PUBLICATION_UPDATE_ON_OK
......
......@@ -88,6 +88,7 @@
"Can't delete this record since several publications refer to it.": 'Impossible de détruire cet enregistrement car des publications lui font référence.',
"Can't insert the article.": 'Impossible de créer cet article.',
"Can't insert the harvester.": 'Impossible de créer ce moissonneur.',
"Can't insert the harvester. ": "Impossible de créer ce moissonneur. ",
"Can't insert the report.": 'Impossible de créer ce rapport.',
"Can't insert the talk/proceeding.": 'Impossible de créer cet acte ou présentation.',
"Can't updated a publication marked OK.": "Impossible d'actualiser une publication marquée OK.",
......@@ -293,6 +294,7 @@
'HTTP Error': 'HTTP Error',
'id': 'id',
'Id': 'Id',
'Identical harvester already exists.': 'Ce moissonneur existe dans la base de données.',
'Import/Export': 'Importer/Exporter',
'insert MARCXML': 'insérer MARCXML',
'insert new': 'insert new',
......
......@@ -12,7 +12,7 @@ from callbacks import (INHIBIT_CASCADE_DELETE,
INHIBIT_CONTROLLER_INSERT,
INHIBIT_CONTROLLER_UPDATE,
INHIBIT_DUPLICATE_PUBLICATION,
INHIBIT_HARVESTER_ON_CATEGORY,
INHIBIT_HARVESTER,
INHIBIT_PUBLICATION_DELETE_ON_OK,
INHIBIT_PUBLICATION_UPDATE_ON_OK)
......@@ -31,8 +31,9 @@ from regex import (REG_COLLABORATION,
#-------------------------------------------------------------------------------
plugins = PluginManager()
plugins.dbui.app_css = 'static/my.css'
# plugins.dbui.app_js_dir = None
# plugins.dbui.app_libmin = None
plugins.dbui.app_debug = None
plugins.dbui.app_lg = 'static/track_publications/locale/trp-lang-fr.js'
plugins.dbui.app_libmin = 'static/track_publications-min.js'
plugins.dbui.app_script = 'static/app.js'
# plugins.dbui.app_script_dir = None
......
......@@ -24,7 +24,7 @@ db.define_table("harvesters",
Field("scan", "boolean", comment=tp_scan, default=False),
migrate="harvesters.table")
db.harvesters._before_insert.append(INHIBIT_HARVESTER_ON_CATEGORY)
db.harvesters._before_insert.append(INHIBIT_HARVESTER)
db.harvesters.collections.filter_in = dbui.CLEAN_SPACES
db.harvesters.controller.filter_in = dbui.CLEAN_SPACES
......
......@@ -54,6 +54,9 @@ gridModifier.append_filter(('id_categories', '==', T('is equal to')))
gridModifier.configure_filters(plugins=['pFormToolTip'], width=300)
gridModifier.configure_gridWithFilter(selectorTitle=T('Filter'))
gridModifier.configure(plugins=[{'ptype': 'pGridRowEditorAddWizard',
'wizard' :{'xtype': 'xwizardharvester'}}])
#-------------------------------------------------------------------------------
#
# STORE CONFIGURATiON
......
......@@ -187,12 +187,15 @@ def INHIBIT_DUPLICATE_PUBLICATION(publication):
return False
def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
"""Inhibit the insert of similar harvesters.
def INHIBIT_HARVESTER(harvester):
"""Inhibit the insert of similar harvesters:
For a project, one automaton can only proceed publication
of the same code, e.g ACL or ACLN but not both, but automatons
can scan different stores.
* For a project, one automaton can only proceed publication
of the same code, e.g ACL or ACLN but not both, but automatons
can scan different stores.
* Reject identical harvester, same project, controller, store,
collection and category.
Args:
harvester (dict): harvester fields passed to insert.
......@@ -209,7 +212,7 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
id_projects=harvester['id_projects'],
id_teams=harvester['id_teams'])
if not id_harvester:
if id_harvester is None:
return False
# for a project, one automaton can only proceed publication
......@@ -226,6 +229,23 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
return True
# duplicate harvesters
id_harvester = get_id(db.harvesters,
collections=harvester['collections'],
controller=harvester['controller'],
host=harvester['host'],
id_categories=harvester['id_categories'],
id_projects=harvester['id_projects'],
id_teams=harvester['id_teams'])
if id_harvester is not None:
db.harvesters[CALLBACK_ERRORS] = [
T("Can't insert the harvester. "),
T("Identical harvester already exists.")]
return True
return False
......
......@@ -182,7 +182,7 @@ def search_synonym(table, fieldname, value, create=False):
return id_rec
# nothing found, have a look to the synonyms field
query = table.synonyms.contains(value)
query = table.synonyms.contains(value.encode('utf-8'))
setrows = db(query)
# no synonym found, create the entry
......
......@@ -60,10 +60,20 @@
MathJax
</a>
</li>
<li class="my-li">
<a href="http://matplotlib.org/" target="_blank">
Matplotlib
</a>
</li>
<li class="my-li">
<a href="https://marprod.in2p3.fr/plugin_dbui_book" target="_blank">
plugin dbui
</a>
</li>
<li class="my-li">
<a href="http://pandas.pydata.org/" target="_blank">
Pandas
</a>
</li>
<li class="my-li">
<a href="http://www.python.org/" target="_blank">
......@@ -88,6 +98,6 @@
<p class="my-p">Ce logiciel est distribué sous la licence opence source
<a href="http://www.cecill.info/" target="_blank">CeCILL</a>.</p>
<p class="my-p">&copy; 2010 - 2014 Renaud Le Gac</p>
<p class="my-p">&copy; 2010 - 2015 Renaud Le Gac</p>
</body>
</html>
\ No newline at end of file
......@@ -2,13 +2,17 @@
HEAD
0.9.3 (Dec 2015)
- Require plugin_dbui 0.7.3
- Add the wizard to create an harvester.
0.9.2 (Nov 2015)
- Require plugin_dbui 0.7.2 or later release.
- Major review of the user guide.
- Protect the table controllers against duplicate pair (harvester, category)
as well as duplicate usage of the publication category.
- Several bugs fixed.
- Fix bug in the list extraction to CSF file.
- Fix bug in the list extraction to CSV file.
- Fix a bug in the graph generation when the database is empty.
0.9.0 (Nov 2015)
......
......@@ -10,18 +10,23 @@ if (App.debug) {
enabled: true,
paths: {
'App': '/' + App.name + '/static/plugin_dbui/src',
'Ext': '/' + App.name + '/static/plugin_extjs/src'
'Ext': '/' + App.name + '/static/plugin_extjs/src',
'Trp': '/' + App.name + '/static/track_publications/src'
}
});
}
Ext.require('App.container.Viewport');
Ext.require('App.grid.plugin.RowEditorAddWizard');
Ext.require('App.plugin.MathJax');
Ext.require('Ext.direct.Manager');
Ext.require('Ext.direct.RemotingProvider');
Ext.require('Ext.EventManager');
Ext.require('Ext.tip.QuickTipManager');
Ext.require('Trp.wizard.Harvester');
Ext.onReady(function(){
"use strict";
......