Merge branch '37-wizard-create-harvester' into 'master'

37 wizard create harvester

* Add the wizard to create an harvester.
* Require `plugin_dbui` release 0.7.3.
* Base on the javascript class `Wizard` and on the grid plugin `RowEditorAddWizard`.
* Introduce the javascript namesapce `Trp`.
* Add the javascript library `track_publications`.
* Close #37

See merge request !48

.gitignore

@@ -23,6 +23,6 @@ static/plugin_ace
 static/plugin_dbui
 static/plugin_extjs
 static/plugin_mathjax
-static/track-publication*.js
+static/track_publications*.js
 views/plugin_dbui
 controllers/plugin_dbui.py

build_version.py

@@ -36,6 +36,7 @@ from subprocess import call
 APP = os.path.basename(os.getcwd())
 API = 'api'
 CHANGELOG = 'static/CHANGELOG'
+DBUISRC = 'static/plugin_dbui/src'
 DOCS = 'static/docs'
 DOCSRC = 'docs'
 EXTJSSRC = 'static/plugin_extjs/src'
@@ -202,22 +203,20 @@ def compile_js():
             print 'Remove old javascript library', item
 
     # debug version of the javascript library
-    cwd = os.getcwd()
-
-    cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
-           "compile", "-class", opj(cwd, JSLIBSRC),
-           "exclude", "--namespace", "Ext",
-           "and", "concat", opj(cwd, JSLIBDEBUG)]
+    cmd = ["sencha", "-sdk", EXTJSSRC,
+           "compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
+           "exclude", "--namespace", "Ext,App",
+           "and", "concat", JSLIBDEBUG]
 
     call(cmd)
 
     print 'Debug version of the javascript library', JSLIBDEBUG, 'is ready'
 
     # Minified version of the javascript library
-    cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
-           "compile", "-class", opj(cwd, JSLIBSRC),
-           "exclude", "--namespace", "Ext",
-           "and", "concat", "--yui", opj(cwd, JSLIBMIN)]
+    cmd = ["sencha", "-sdk", EXTJSSRC,
+           "compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
+           "exclude", "--namespace", "Ext,App",
+           "and", "concat", "--yui", JSLIBMIN]
 
     call(cmd)
 
@@ -419,7 +418,9 @@ if __name__ == '__main__':
 
     APS.add_argument("-C", "--commit-changelog",
                      action="store_true",
-                     help="commit CHANGELOG.")
+                     help="commit CHANGELOG. "
+                          "To be used with --write-release. "
+                          "Recommend to use --start-release.")
 
     APS.add_argument("-j", "--jsduck",
                      action="store_true",
@@ -435,7 +436,11 @@ if __name__ == '__main__':
 
     APS.add_argument("-s", "--start-release",
                      action="store_true",
-                     help="start the new release.")
+                     help="start the new release cycle. "
+                          "Set the release number in the changelog and "
+                           "commit changes in the master branch. "
+                           "Equivalent to --write-release followed by "
+                           "--commit-changelog.")
 
     APS.add_argument("-u", "--user-doc",
                      action="store_true",
@@ -451,7 +456,9 @@ if __name__ == '__main__':
 
     APS.add_argument("-w", "--write-release",
                      action="store_true",
-                     help="write the release number in CHANGELOG.")
+                     help="write the release number in CHANGELOG. "
+                          "To be used with --commit-changelog. "
+                          "Recommend to use --start-release.")
 
     ARGS = APS.parse_args()
 

controllers/wizards.py

@@ -8,8 +8,10 @@ import re
 from check_tools import check_publication
 from gluon.storage import Storage
 from harvest_tools import DRY_RUN
-from plugin_dbui import (is_foreign_field,
+from plugin_dbui import (CALLBACK_ERRORS,
+                         is_foreign_field,
                          get_foreign_field,
+                         get_id,
                          Selector,
                          to_fields)
 
@@ -28,7 +30,7 @@ def check_validate():
     """
     counters = {}
     logs = []
-    id_ok = db(db.status.code=='OK').select().first().id
+    id_ok = db(db.status.code == 'OK').select().first().id
 
     # get user requirement
     selector = Selector(virtdb.check_selector, exclude_fields=('mode'))
@@ -188,4 +190,116 @@ def extract_authors():
     if not case_1:
         return MSG_NO_AUTHORS
 
-    return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
\ No newline at end of file
+    return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
+
+
+def harvester():
+    """Process the data send by the wizard harvester and fill the
+    harvesters table. The data block contains the keys:
+
+        * automaton (str)
+        * collaboration (str)
+        * collection (str)
+        * people (list)
+        * producer (str) either 'collaboration' or 'people'
+        * project (int)
+        * store (str) either cds.cern.ch or inspirehep.net
+
+    The logic to interpret the data depends on the store.
+
+    """
+    # shortcuts
+    vars = request.vars
+    automaton = vars.automaton
+    organisation = db.organisation
+    project = vars.project
+    store = vars.store
+
+    # protection
+    if store not in ("cds.cern.ch", "inspirehep.net"):
+        raise HTTP(500, "Invalid store !")
+
+    values = Storage(controller=automaton,
+                     host=store,
+                     id_projects=project)
+
+    # the team
+    rec_id = get_id(organisation, id_projects=project)
+    if rec_id is None:
+        raise HTTP(500, "Project is unknown !")
+
+    values.id_teams = organisation[rec_id].id_teams
+
+    # the collection
+    if store == "cds.cern.ch":
+        values.collections = vars.collection
+
+    elif store == "inspirehep.net":
+        collection = []
+
+        collaboration = vars.collaboration
+        if collaboration:
+            collection.append("cn %s" % collaboration)
+
+        else:
+            collection.append("(a %s)" % " or ".join(vars.people))
+
+        if automaton == "articles":
+            collection.append("tp p and not tc c")
+
+        elif automaton == "proceedings":
+            collection.append("tp p and tc c")
+
+        values.collections = "find %s" % " and ".join(collection)
+
+    # the publication category
+    if automaton == "articles":
+        id_category = get_id(db.categories, code="ACL")
+
+    elif automaton in ("notes", "reports"):
+        id_category = get_id(db.categories, code="AP")
+
+    elif automaton == "preprints":
+        id_category = get_id(db.categories, code="PRE")
+
+    elif automaton == "proceedings":
+        id_category = get_id(db.categories, code="ACTI")
+
+    elif automaton == "talks":
+        id_category = get_id(db.categories, code="COM")
+
+    elif automaton == "theses":
+        id_category = get_id(db.categories, code="PHD")
+
+    if id_category is None:
+        raise HTTP(500, "Category is unknown !")
+
+    values.id_categories = id_category
+
+    # insert new values in the database
+    msg = None
+    try:
+        rec_id = db.harvesters.insert(**values)
+        if rec_id:
+            return
+
+        # operation can be reject by callback table._before_insert
+        else:
+            msg = "Fail to insert the new harvester in the database."
+            if CALLBACK_ERRORS in db.harvesters:
+                msg = db.harvesters._callback_errors
+
+            # reduce the error message
+            if isinstance(msg, list):
+                msg = "%s %s" % (msg[0], msg[-1])
+
+    # operation is rejected by the database
+    except Exception as dbe:
+        raise HTTP(500, dbe.message)
+
+    # operation is rejected by the callback
+    # NOTE in the else branch to avoid recursive exception generation
+    if msg is not None:
+        raise HTTP(500, msg)
+
+    return

languages/fr-fr.py

@@ -88,6 +88,7 @@
 "Can't delete this record since several publications refer to it.": 'Impossible de détruire cet enregistrement car des publications lui font référence.',
 "Can't insert the article.": 'Impossible de créer cet article.',
 "Can't insert the harvester.": 'Impossible de créer ce moissonneur.',
+"Can't insert the harvester. ": "Impossible de créer ce moissonneur. ",
 "Can't insert the report.": 'Impossible de créer ce rapport.',
 "Can't insert the talk/proceeding.": 'Impossible de créer cet acte ou présentation.',
 "Can't updated a publication marked OK.": "Impossible d'actualiser une publication marquée OK.",
@@ -293,6 +294,7 @@
 'HTTP Error': 'HTTP Error',
 'id': 'id',
 'Id': 'Id',
+'Identical harvester already exists.': 'Ce moissonneur existe dans la base de données.',
 'Import/Export': 'Importer/Exporter',
 'insert MARCXML': 'insérer MARCXML',
 'insert new': 'insert new',

models/common_settings.py

@@ -12,7 +12,7 @@ from callbacks import (INHIBIT_CASCADE_DELETE,
                        INHIBIT_CONTROLLER_INSERT,
                        INHIBIT_CONTROLLER_UPDATE,
                        INHIBIT_DUPLICATE_PUBLICATION,
-                       INHIBIT_HARVESTER_ON_CATEGORY,
+                       INHIBIT_HARVESTER,
                        INHIBIT_PUBLICATION_DELETE_ON_OK,
                        INHIBIT_PUBLICATION_UPDATE_ON_OK)
 
@@ -31,8 +31,9 @@ from regex import (REG_COLLABORATION,
 #-------------------------------------------------------------------------------
 plugins = PluginManager()
 plugins.dbui.app_css = 'static/my.css'
-# plugins.dbui.app_js_dir = None
-# plugins.dbui.app_libmin = None
+plugins.dbui.app_debug = None
+plugins.dbui.app_lg = 'static/track_publications/locale/trp-lang-fr.js'
+plugins.dbui.app_libmin = 'static/track_publications-min.js'
 plugins.dbui.app_script = 'static/app.js'
 # plugins.dbui.app_script_dir = None
 

models/db1_harvesters.py

@@ -24,7 +24,7 @@ db.define_table("harvesters",
     Field("scan", "boolean", comment=tp_scan, default=False),
     migrate="harvesters.table")
 
-db.harvesters._before_insert.append(INHIBIT_HARVESTER_ON_CATEGORY)
+db.harvesters._before_insert.append(INHIBIT_HARVESTER)
 
 db.harvesters.collections.filter_in = dbui.CLEAN_SPACES
 db.harvesters.controller.filter_in = dbui.CLEAN_SPACES

models/ui_harvesters.py

@@ -54,6 +54,9 @@ gridModifier.append_filter(('id_categories', '==', T('is equal to')))
 gridModifier.configure_filters(plugins=['pFormToolTip'], width=300)
 gridModifier.configure_gridWithFilter(selectorTitle=T('Filter'))
 
+gridModifier.configure(plugins=[{'ptype': 'pGridRowEditorAddWizard',
+                                 'wizard' :{'xtype': 'xwizardharvester'}}])
+
 #-------------------------------------------------------------------------------
 #
 # STORE CONFIGURATiON

modules/callbacks.py

@@ -187,12 +187,15 @@ def INHIBIT_DUPLICATE_PUBLICATION(publication):
     return False
 
 
-def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
-    """Inhibit the insert of similar harvesters.
+def INHIBIT_HARVESTER(harvester):
+    """Inhibit the insert of similar harvesters:
 
-    For a project, one automaton can only proceed publication
-    of the same code, e.g ACL or ACLN but not both, but automatons
-    can scan different stores.
+        * For a project, one automaton can only proceed publication
+          of the same code, e.g ACL or ACLN but not both, but automatons
+          can scan different stores.
+
+        * Reject identical harvester, same project, controller, store,
+          collection and category.
 
     Args:
         harvester (dict): harvester fields passed to insert.
@@ -209,7 +212,7 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
                           id_projects=harvester['id_projects'],
                           id_teams=harvester['id_teams'])
 
-    if not id_harvester:
+    if id_harvester is None:
         return False
 
     # for a project, one automaton can only proceed publication
@@ -226,6 +229,23 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
 
         return True
 
+    # duplicate harvesters
+    id_harvester = get_id(db.harvesters,
+                          collections=harvester['collections'],
+                          controller=harvester['controller'],
+                          host=harvester['host'],
+                          id_categories=harvester['id_categories'],
+                          id_projects=harvester['id_projects'],
+                          id_teams=harvester['id_teams'])
+
+    if id_harvester is not None:
+
+        db.harvesters[CALLBACK_ERRORS] = [
+            T("Can't insert the harvester. "),
+            T("Identical harvester already exists.")]
+
+        return True
+
     return False
 
 

static/app.js

@@ -10,18 +10,23 @@ if (App.debug) {
         enabled: true,
         paths: {
             'App': '/' + App.name + '/static/plugin_dbui/src',
-            'Ext': '/' + App.name + '/static/plugin_extjs/src'
+            'Ext': '/' + App.name + '/static/plugin_extjs/src',
+            'Trp': '/' + App.name + '/static/track_publications/src'
         }
     });
 }
 
 Ext.require('App.container.Viewport');
+Ext.require('App.grid.plugin.RowEditorAddWizard');
 Ext.require('App.plugin.MathJax');
+
 Ext.require('Ext.direct.Manager');
 Ext.require('Ext.direct.RemotingProvider');
 Ext.require('Ext.EventManager');
 Ext.require('Ext.tip.QuickTipManager');
 
+Ext.require('Trp.wizard.Harvester');
+
 Ext.onReady(function(){
 
     "use strict";

static/track_publications/locale/trp-lang-en.js

+/**
+ * Mother language
+ *
+ */

static/track_publications/locale/trp-lang-fr.js

+/**
+ * French translation
+ *
+ */
+Ext.define('Trp.local.frwizard.Harvester', {
+    override: 'Trp.wizard.Harvester',
+
+    textArticle: "article",
+    textNote: "note",
+    textPreprint: "preprint",
+    textProceeding: "acte de conférence",
+    textReport: "rapport",
+    textTalk: "présentation orale",
+    textThesis: "mémoire",
+
+    textCategory: [
+        "<h4> 3/7 catégorie</h4>",
+        "<p>Sélectionner la <i>catégorie de publication</i>",
+        "à chercher dans l'entrepôt:</p>"
+    ],
+
+    textCollection1: [
+        "<h4> 6/7 collection</h4>",
+        "<p>Entrer le non de la collection:</p>"
+    ],
+
+    textCollection2: [
+        "<p>Les collections par expérience, connues de <i>cds.cern.ch</i>,",
+        "sont consutables à",
+        "<a href='https://cds.cern.ch/collection/CERN Experiments'",
+        "target='_blank'>",
+        "https://cds.cern.ch/collection/CERN Experiments</a>.</p>"
+    ],
+
+    textCollaboration1: [
+        "<h4> 5/7 collaboration</h4>",
+        "<p>Entrer le non de la collaboration:</p>"
+    ],
+
+    textCollaboration2:[
+        "<p>Les collaborations connues de <i>inspirehep.net</i>,",
+        "sont consultables à ",
+        "<a href='https://inspirehep.net/collection/Experiments'",
+        "target='_blank'>",
+        "https://inspirehep.net/collection/Experiments</a>.</p>"
+    ],
+
+    textPeople1: [
+        "<h4> 5/7 personne</h4>",
+        "<p>Entrer le nom des personnes.<br>",
+        "Un nom par ligne.<br>",
+        "Les valeurs possibles sont: <i>John Doe</i> or <i>j doe</i> or <i>doe</i>.",
+        "</p>"
+    ],
+
+    textPeople2: [
+        "<p>Ajouter plus d'éléments à la liste en cliquant sur son menu ",
+        "contextuelle (clic droit).</p>"
+    ],
+
+    textProducer: [
+        "<h4> 4/7 producteur</h4>",
+        "Les publications sont produites par:"
+    ],
+
+    textProducerCollaboration: "Une collaboration internationale / expérience",
+
+    textProducerPeople: "Des personnes non membre d'une collaboration",
+
+    textProject1: [
+        "<h4> 6/7 projet</h4>",
+        "<p>Associer les publications trouvées à mon projet:</p>"
+    ],
+
+    textProject2: [
+        "<p>Sauver la configuration en cliquant sur le bouton <i>Fin</i>.</p>"
+    ],
+
+    textStore: [
+        "<h4> 2/7 entrepôt</h4>",
+        "<p>L'entrepôt <i>inspirehep.net</i> contient les <i>articles</i>",
+        "les <i>actes de conférence</i> pour la plus part des collaborations ",
+        "ou groupe de personne.</p>",
+        "<p>L'entrepôt <i>cds.cern.ch</i> contient toutes les catégories",
+        "de publications pour les expériences installées aux CERN.</p>"
+    ],
+
+    textWelcome: [
+        "<h4> 1/7 création d'un moissonneur</h4>",
+        "<p>Le moissonneur sélectionne des publications dans l'entrepôt",
+        "<i>inspirehep.net</i> ou <i>cds.cern.ch</i> et les associe à l'un",
+        "de vos projets.</p>",
+        "<p>Cliquer sur le boutton <i>suivant</i> ou <i>Précédent</i> pour",
+        "changer de page.",
+        "A la fin, cliquer sur le boutton <i>Fin</i>.</p>",
+        "<p>Le boutton <i>Fin</i> est actif",
+        "quand tous les champs sont remplis.</p>"
+    ]
+});

static/track_publications/src/wizard/Harvester.js

+/**
+ * Wizard to create an harvester.
+ *
+ * @since 0.9.3
+ *
+ */
+Ext.define('Trp.wizard.Harvester', {
+
+    extend: 'App.wizard.Wizard',
+    alias: 'widget.xwizardharvester',
+    uses: [
+        'App.form.field.ComboBox',
+        'Ext.form.RadioGroup',
+        'Ext.form.field.Text'
+    ],
+
+    url: 'wizards/harvester',
+
+    // private properties for internationalisation
+    textArticle: "article",
+    textNote: "note",
+    textPreprint: "preprint",
+    textProceeding: "proceeding",
+    textReport: "report",
+    textTalk: "talk",
+    textThesis: "thesis",
+
+    textCategory: [
+        "<h4> 3/7 category</h4>",
+        "<p>Select the <i>category of publication</i> you are looking for:</p>"
+    ],
+
+    textCollection1: [
+        "<h4> 6/7 collection</h4>",
+        "<p>Enter the name of the collection:</p>"
+    ],
+
+    textCollection2: [
+        "<p>Collections per experiment, know by <i>cds.cern.ch</i>,",
+        "can be found at",
+        "<a href='https://cds.cern.ch/collection/CERN Experiments'",
+        "target='_blank'>",
+        "https://cds.cern.ch/collection/CERN Experiments</a>.</p>"
+    ],
+
+    textCollaboration1: [
+        "<h4> 5/7 collaboration</h4>",
+        "<p>Enter the name of the collaboration:</p>"
+    ],
+
+    textCollaboration2: [
+        "<p>Collaborations, know by <i>inspirehep.net</i>,",
+        "can be found at",
+        "<a href='https://inspirehep.net/collection/Experiments'",
+        "target='_blank'>",
+        "https://inspirehep.net/collection/Experiments</a>.</p>"
+    ],
+
+    textPeople1: [
+        "<h4> 5/7 people</h4>",
+        "<p>Enter names of the people.<br>",
+        "One name per line.<br>",
+        "Possible values are: <i>John Doe</i> or <i>j doe</i> or <i>doe</i>.",
+        "</p>"
+    ],
+
+    textPeople2: [
+        "<p>Add more entries in the list by launching its contextual menu",
+        "(right click on the list widget).</p>"
+    ],
+
+    textProducer: [
+        "<h4> 4/7 producer</h4>",
+        "Publications are produced by:"
+    ],
+
+    textProducerCollaboration: 'International collaboration / experiment',
+
+    textProducerPeople: 'People not belonging to a collaboration',
+
+    textProject1: [
+        "<h4> 6/7 project</h4>",
+        "<p>Associate the found publications to my project:</p>"
+    ],
+
+    textProject2: [
+        "<p>Save the configuration by clicking the button <i>Finish</i>.</p>"
+    ],
+
+    textStore: [
+        "<h4> 2/7 store</h4>",
+        "<p>The store <i>inspirehep.net</i> allows to select <i>articles</i>",
+        "and <i>proceedings</i> for almost all collaborations or group of people.</p>",
+        "<p>The store <i>cds.cern.ch</i> allows to select almost all categories",
+        "of publications for the experiments running at CERN.</p>"
+    ],
+
+    textWelcome: [
+        "<h4> 1/7 harvester creation</h4>",
+        "<p>The harvester selects publications in the <i>inspirehep.net</i>",
+        "or <i>cds.cern.ch</i> stores",
+        "and it associates them to one of your projects.</p>",
+        "<p>Click on the button <i>Next</i> or <i>Previous</i> to navigate",
+        "between the wizard pages.",
+        "At the end click on the button <i>Finish</i>.</p>",
+        "<p>The <i>Finish</i> button is enable ",
+        "when all fields are defined.</p>"
+    ],
+
+    // jshint strict: false
+
+    // private method requires by the Ext JS component model
+    initComponent: function () {
+
+        var me = this;
+
+        //initialise the base class
+        me.callParent(arguments);
+
+        // add the pages
+        me.add([
+            me.pageWelcome(),
+            me.pageStore(),
+            me.pageCategory(),
+            me.pageProducer(),
+            me.pageCollaborationOrPeople(),
+            me.pageCollection(),
+            me.pageProject()
+        ]);
+
+        // handlers
+        me.down('#fieldProducer').on('change', me.onProducerChange, me);
+        me.down('#fieldStore').on('change', me.onStoreChange, me);
+
+        // set the default store and trigger the handler onStoreChange
+        me.down('#fieldStore').setValue('inspirehep.net');
+    },
+
+    // jshint strict: true
+
+    /**
+     * Handler calls when the producer of publications is selected.
+     * It is required using the store inspirehep.net.
+     * It enables the page collaboration or people.
+     *
+     * @param {Ext.form.RadioGroup} radiofield
+     *
+     */
+    onProducerChange: function (radioField) {
+
+        "