Commit 70e6d521 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch '37-wizard-create-harvester' into 'master'

37 wizard create harvester

* Add the wizard to create an harvester.
* Require `plugin_dbui` release 0.7.3.
* Base on the javascript class `Wizard` and on the grid plugin `RowEditorAddWizard`.
* Introduce the javascript namesapce `Trp`.
* Add the javascript library `track_publications`.
* Close #37

See merge request !48
parents 2737167b e0f1cdcd
......@@ -23,6 +23,6 @@ static/plugin_ace
static/plugin_dbui
static/plugin_extjs
static/plugin_mathjax
static/track-publication*.js
static/track_publications*.js
views/plugin_dbui
controllers/plugin_dbui.py
......@@ -36,6 +36,7 @@ from subprocess import call
APP = os.path.basename(os.getcwd())
API = 'api'
CHANGELOG = 'static/CHANGELOG'
DBUISRC = 'static/plugin_dbui/src'
DOCS = 'static/docs'
DOCSRC = 'docs'
EXTJSSRC = 'static/plugin_extjs/src'
......@@ -202,22 +203,20 @@ def compile_js():
print 'Remove old javascript library', item
# debug version of the javascript library
cwd = os.getcwd()
cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
"compile", "-class", opj(cwd, JSLIBSRC),
"exclude", "--namespace", "Ext",
"and", "concat", opj(cwd, JSLIBDEBUG)]
cmd = ["sencha", "-sdk", EXTJSSRC,
"compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
"exclude", "--namespace", "Ext,App",
"and", "concat", JSLIBDEBUG]
call(cmd)
print 'Debug version of the javascript library', JSLIBDEBUG, 'is ready'
# Minified version of the javascript library
cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
"compile", "-class", opj(cwd, JSLIBSRC),
"exclude", "--namespace", "Ext",
"and", "concat", "--yui", opj(cwd, JSLIBMIN)]
cmd = ["sencha", "-sdk", EXTJSSRC,
"compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
"exclude", "--namespace", "Ext,App",
"and", "concat", "--yui", JSLIBMIN]
call(cmd)
......@@ -419,7 +418,9 @@ if __name__ == '__main__':
APS.add_argument("-C", "--commit-changelog",
action="store_true",
help="commit CHANGELOG.")
help="commit CHANGELOG. "
"To be used with --write-release. "
"Recommend to use --start-release.")
APS.add_argument("-j", "--jsduck",
action="store_true",
......@@ -435,7 +436,11 @@ if __name__ == '__main__':
APS.add_argument("-s", "--start-release",
action="store_true",
help="start the new release.")
help="start the new release cycle. "
"Set the release number in the changelog and "
"commit changes in the master branch. "
"Equivalent to --write-release followed by "
"--commit-changelog.")
APS.add_argument("-u", "--user-doc",
action="store_true",
......@@ -451,7 +456,9 @@ if __name__ == '__main__':
APS.add_argument("-w", "--write-release",
action="store_true",
help="write the release number in CHANGELOG.")
help="write the release number in CHANGELOG. "
"To be used with --commit-changelog. "
"Recommend to use --start-release.")
ARGS = APS.parse_args()
......
......@@ -8,8 +8,10 @@ import re
from check_tools import check_publication
from gluon.storage import Storage
from harvest_tools import DRY_RUN
from plugin_dbui import (is_foreign_field,
from plugin_dbui import (CALLBACK_ERRORS,
is_foreign_field,
get_foreign_field,
get_id,
Selector,
to_fields)
......@@ -28,7 +30,7 @@ def check_validate():
"""
counters = {}
logs = []
id_ok = db(db.status.code=='OK').select().first().id
id_ok = db(db.status.code == 'OK').select().first().id
# get user requirement
selector = Selector(virtdb.check_selector, exclude_fields=('mode'))
......@@ -188,4 +190,116 @@ def extract_authors():
if not case_1:
return MSG_NO_AUTHORS
return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
\ No newline at end of file
return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
def harvester():
"""Process the data send by the wizard harvester and fill the
harvesters table. The data block contains the keys:
* automaton (str)
* collaboration (str)
* collection (str)
* people (list)
* producer (str) either 'collaboration' or 'people'
* project (int)
* store (str) either cds.cern.ch or inspirehep.net
The logic to interpret the data depends on the store.
"""
# shortcuts
vars = request.vars
automaton = vars.automaton
organisation = db.organisation
project = vars.project
store = vars.store
# protection
if store not in ("cds.cern.ch", "inspirehep.net"):
raise HTTP(500, "Invalid store !")
values = Storage(controller=automaton,
host=store,
id_projects=project)
# the team
rec_id = get_id(organisation, id_projects=project)
if rec_id is None:
raise HTTP(500, "Project is unknown !")
values.id_teams = organisation[rec_id].id_teams
# the collection
if store == "cds.cern.ch":
values.collections = vars.collection
elif store == "inspirehep.net":
collection = []
collaboration = vars.collaboration
if collaboration:
collection.append("cn %s" % collaboration)
else:
collection.append("(a %s)" % " or ".join(vars.people))
if automaton == "articles":
collection.append("tp p and not tc c")
elif automaton == "proceedings":
collection.append("tp p and tc c")
values.collections = "find %s" % " and ".join(collection)
# the publication category
if automaton == "articles":
id_category = get_id(db.categories, code="ACL")
elif automaton in ("notes", "reports"):
id_category = get_id(db.categories, code="AP")
elif automaton == "preprints":
id_category = get_id(db.categories, code="PRE")
elif automaton == "proceedings":
id_category = get_id(db.categories, code="ACTI")
elif automaton == "talks":
id_category = get_id(db.categories, code="COM")
elif automaton == "theses":
id_category = get_id(db.categories, code="PHD")
if id_category is None:
raise HTTP(500, "Category is unknown !")
values.id_categories = id_category
# insert new values in the database
msg = None
try:
rec_id = db.harvesters.insert(**values)
if rec_id:
return
# operation can be reject by callback table._before_insert
else:
msg = "Fail to insert the new harvester in the database."
if CALLBACK_ERRORS in db.harvesters:
msg = db.harvesters._callback_errors
# reduce the error message
if isinstance(msg, list):
msg = "%s %s" % (msg[0], msg[-1])
# operation is rejected by the database
except Exception as dbe:
raise HTTP(500, dbe.message)
# operation is rejected by the callback
# NOTE in the else branch to avoid recursive exception generation
if msg is not None:
raise HTTP(500, msg)
return
......@@ -88,6 +88,7 @@
"Can't delete this record since several publications refer to it.": 'Impossible de détruire cet enregistrement car des publications lui font référence.',
"Can't insert the article.": 'Impossible de créer cet article.',
"Can't insert the harvester.": 'Impossible de créer ce moissonneur.',
"Can't insert the harvester. ": "Impossible de créer ce moissonneur. ",
"Can't insert the report.": 'Impossible de créer ce rapport.',
"Can't insert the talk/proceeding.": 'Impossible de créer cet acte ou présentation.',
"Can't updated a publication marked OK.": "Impossible d'actualiser une publication marquée OK.",
......@@ -293,6 +294,7 @@
'HTTP Error': 'HTTP Error',
'id': 'id',
'Id': 'Id',
'Identical harvester already exists.': 'Ce moissonneur existe dans la base de données.',
'Import/Export': 'Importer/Exporter',
'insert MARCXML': 'insérer MARCXML',
'insert new': 'insert new',
......
......@@ -12,7 +12,7 @@ from callbacks import (INHIBIT_CASCADE_DELETE,
INHIBIT_CONTROLLER_INSERT,
INHIBIT_CONTROLLER_UPDATE,
INHIBIT_DUPLICATE_PUBLICATION,
INHIBIT_HARVESTER_ON_CATEGORY,
INHIBIT_HARVESTER,
INHIBIT_PUBLICATION_DELETE_ON_OK,
INHIBIT_PUBLICATION_UPDATE_ON_OK)
......@@ -31,8 +31,9 @@ from regex import (REG_COLLABORATION,
#-------------------------------------------------------------------------------
plugins = PluginManager()
plugins.dbui.app_css = 'static/my.css'
# plugins.dbui.app_js_dir = None
# plugins.dbui.app_libmin = None
plugins.dbui.app_debug = None
plugins.dbui.app_lg = 'static/track_publications/locale/trp-lang-fr.js'
plugins.dbui.app_libmin = 'static/track_publications-min.js'
plugins.dbui.app_script = 'static/app.js'
# plugins.dbui.app_script_dir = None
......
......@@ -24,7 +24,7 @@ db.define_table("harvesters",
Field("scan", "boolean", comment=tp_scan, default=False),
migrate="harvesters.table")
db.harvesters._before_insert.append(INHIBIT_HARVESTER_ON_CATEGORY)
db.harvesters._before_insert.append(INHIBIT_HARVESTER)
db.harvesters.collections.filter_in = dbui.CLEAN_SPACES
db.harvesters.controller.filter_in = dbui.CLEAN_SPACES
......
......@@ -54,6 +54,9 @@ gridModifier.append_filter(('id_categories', '==', T('is equal to')))
gridModifier.configure_filters(plugins=['pFormToolTip'], width=300)
gridModifier.configure_gridWithFilter(selectorTitle=T('Filter'))
gridModifier.configure(plugins=[{'ptype': 'pGridRowEditorAddWizard',
'wizard' :{'xtype': 'xwizardharvester'}}])
#-------------------------------------------------------------------------------
#
# STORE CONFIGURATiON
......
......@@ -187,12 +187,15 @@ def INHIBIT_DUPLICATE_PUBLICATION(publication):
return False
def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
"""Inhibit the insert of similar harvesters.
def INHIBIT_HARVESTER(harvester):
"""Inhibit the insert of similar harvesters:
For a project, one automaton can only proceed publication
of the same code, e.g ACL or ACLN but not both, but automatons
can scan different stores.
* For a project, one automaton can only proceed publication
of the same code, e.g ACL or ACLN but not both, but automatons
can scan different stores.
* Reject identical harvester, same project, controller, store,
collection and category.
Args:
harvester (dict): harvester fields passed to insert.
......@@ -209,7 +212,7 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
id_projects=harvester['id_projects'],
id_teams=harvester['id_teams'])
if not id_harvester:
if id_harvester is None:
return False
# for a project, one automaton can only proceed publication
......@@ -226,6 +229,23 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
return True
# duplicate harvesters
id_harvester = get_id(db.harvesters,
collections=harvester['collections'],
controller=harvester['controller'],
host=harvester['host'],
id_categories=harvester['id_categories'],
id_projects=harvester['id_projects'],
id_teams=harvester['id_teams'])
if id_harvester is not None:
db.harvesters[CALLBACK_ERRORS] = [
T("Can't insert the harvester. "),
T("Identical harvester already exists.")]
return True
return False
......
......@@ -10,18 +10,23 @@ if (App.debug) {
enabled: true,
paths: {
'App': '/' + App.name + '/static/plugin_dbui/src',
'Ext': '/' + App.name + '/static/plugin_extjs/src'
'Ext': '/' + App.name + '/static/plugin_extjs/src',
'Trp': '/' + App.name + '/static/track_publications/src'
}
});
}
Ext.require('App.container.Viewport');
Ext.require('App.grid.plugin.RowEditorAddWizard');
Ext.require('App.plugin.MathJax');
Ext.require('Ext.direct.Manager');
Ext.require('Ext.direct.RemotingProvider');
Ext.require('Ext.EventManager');
Ext.require('Ext.tip.QuickTipManager');
Ext.require('Trp.wizard.Harvester');
Ext.onReady(function(){
"use strict";
......
/**
* French translation
*
*/
Ext.define('Trp.local.frwizard.Harvester', {
override: 'Trp.wizard.Harvester',
textArticle: "article",
textNote: "note",
textPreprint: "preprint",
textProceeding: "acte de conférence",
textReport: "rapport",
textTalk: "présentation orale",
textThesis: "mémoire",
textCategory: [
"<h4> 3/7 catégorie</h4>",
"<p>Sélectionner la <i>catégorie de publication</i>",
"à chercher dans l'entrepôt:</p>"
],
textCollection1: [
"<h4> 6/7 collection</h4>",
"<p>Entrer le non de la collection:</p>"
],
textCollection2: [
"<p>Les collections par expérience, connues de <i>cds.cern.ch</i>,",
"sont consutables à",
"<a href='https://cds.cern.ch/collection/CERN Experiments'",
"target='_blank'>",
"https://cds.cern.ch/collection/CERN Experiments</a>.</p>"
],
textCollaboration1: [
"<h4> 5/7 collaboration</h4>",
"<p>Entrer le non de la collaboration:</p>"
],
textCollaboration2:[
"<p>Les collaborations connues de <i>inspirehep.net</i>,",
"sont consultables à ",
"<a href='https://inspirehep.net/collection/Experiments'",
"target='_blank'>",
"https://inspirehep.net/collection/Experiments</a>.</p>"
],
textPeople1: [
"<h4> 5/7 personne</h4>",
"<p>Entrer le nom des personnes.<br>",
"Un nom par ligne.<br>",
"Les valeurs possibles sont: <i>John Doe</i> or <i>j doe</i> or <i>doe</i>.",
"</p>"
],
textPeople2: [
"<p>Ajouter plus d'éléments à la liste en cliquant sur son menu ",
"contextuelle (clic droit).</p>"
],
textProducer: [
"<h4> 4/7 producteur</h4>",
"Les publications sont produites par:"
],
textProducerCollaboration: "Une collaboration internationale / expérience",
textProducerPeople: "Des personnes non membre d'une collaboration",
textProject1: [
"<h4> 6/7 projet</h4>",
"<p>Associer les publications trouvées à mon projet:</p>"
],
textProject2: [
"<p>Sauver la configuration en cliquant sur le bouton <i>Fin</i>.</p>"
],
textStore: [
"<h4> 2/7 entrepôt</h4>",
"<p>L'entrepôt <i>inspirehep.net</i> contient les <i>articles</i>",
"les <i>actes de conférence</i> pour la plus part des collaborations ",
"ou groupe de personne.</p>",
"<p>L'entrepôt <i>cds.cern.ch</i> contient toutes les catégories",
"de publications pour les expériences installées aux CERN.</p>"
],
textWelcome: [
"<h4> 1/7 création d'un moissonneur</h4>",
"<p>Le moissonneur sélectionne des publications dans l'entrepôt",
"<i>inspirehep.net</i> ou <i>cds.cern.ch</i> et les associe à l'un",
"de vos projets.</p>",
"<p>Cliquer sur le boutton <i>suivant</i> ou <i>Précédent</i> pour",
"changer de page.",
"A la fin, cliquer sur le boutton <i>Fin</i>.</p>",
"<p>Le boutton <i>Fin</i> est actif",
"quand tous les champs sont remplis.</p>"
]
});
/**
* Wizard to create an harvester.
*
* @since 0.9.3
*
*/
Ext.define('Trp.wizard.Harvester', {
extend: 'App.wizard.Wizard',
alias: 'widget.xwizardharvester',
uses: [
'App.form.field.ComboBox',
'Ext.form.RadioGroup',
'Ext.form.field.Text'
],
url: 'wizards/harvester',
// private properties for internationalisation
textArticle: "article",
textNote: "note",
textPreprint: "preprint",
textProceeding: "proceeding",
textReport: "report",
textTalk: "talk",
textThesis: "thesis",
textCategory: [
"<h4> 3/7 category</h4>",
"<p>Select the <i>category of publication</i> you are looking for:</p>"
],
textCollection1: [
"<h4> 6/7 collection</h4>",
"<p>Enter the name of the collection:</p>"
],
textCollection2: [
"<p>Collections per experiment, know by <i>cds.cern.ch</i>,",
"can be found at",
"<a href='https://cds.cern.ch/collection/CERN Experiments'",
"target='_blank'>",
"https://cds.cern.ch/collection/CERN Experiments</a>.</p>"
],
textCollaboration1: [
"<h4> 5/7 collaboration</h4>",
"<p>Enter the name of the collaboration:</p>"
],
textCollaboration2: [
"<p>Collaborations, know by <i>inspirehep.net</i>,",
"can be found at",
"<a href='https://inspirehep.net/collection/Experiments'",
"target='_blank'>",
"https://inspirehep.net/collection/Experiments</a>.</p>"
],
textPeople1: [
"<h4> 5/7 people</h4>",
"<p>Enter names of the people.<br>",
"One name per line.<br>",
"Possible values are: <i>John Doe</i> or <i>j doe</i> or <i>doe</i>.",
"</p>"
],
textPeople2: [
"<p>Add more entries in the list by launching its contextual menu",
"(right click on the list widget).</p>"
],
textProducer: [
"<h4> 4/7 producer</h4>",
"Publications are produced by:"
],
textProducerCollaboration: 'International collaboration / experiment',
textProducerPeople: 'People not belonging to a collaboration',
textProject1: [
"<h4> 6/7 project</h4>",
"<p>Associate the found publications to my project:</p>"
],
textProject2: [
"<p>Save the configuration by clicking the button <i>Finish</i>.</p>"
],
textStore: [
"<h4> 2/7 store</h4>",
"<p>The store <i>inspirehep.net</i> allows to select <i>articles</i>",
"and <i>proceedings</i> for almost all collaborations or group of people.</p>",
"<p>The store <i>cds.cern.ch</i> allows to select almost all categories",
"of publications for the experiments running at CERN.</p>"
],
textWelcome: [
"<h4> 1/7 harvester creation</h4>",
"<p>The harvester selects publications in the <i>inspirehep.net</i>",
"or <i>cds.cern.ch</i> stores",
"and it associates them to one of your projects.</p>",
"<p>Click on the button <i>Next</i> or <i>Previous</i> to navigate",
"between the wizard pages.",
"At the end click on the button <i>Finish</i>.</p>",
"<p>The <i>Finish</i> button is enable ",
"when all fields are defined.</p>"
],
// jshint strict: false
// private method requires by the Ext JS component model
initComponent: function () {
var me = this;
//initialise the base class
me.callParent(arguments);
// add the pages
me.add([
me.pageWelcome(),
me.pageStore(),
me.pageCategory(),
me.pageProducer(),
me.pageCollaborationOrPeople(),
me.pageCollection(),
me.pageProject()
]);
// handlers
me.down('#fieldProducer').on('change', me.onProducerChange, me);
me.down('#fieldStore').on('change', me.onStoreChange, me);
// set the default store and trigger the handler onStoreChange
me.down('#fieldStore').setValue('inspirehep.net');
},
// jshint strict: true
/**
* Handler calls when the producer of publications is selected.
* It is required using the store inspirehep.net.
* It enables the page collaboration or people.
*
* @param {Ext.form.RadioGroup} radiofield
*
*/
onProducerChange: function (radioField) {
"