Merge branch 'master' into 'production'

Release 0.9.3

* Add mainly the wizard to create harvester
* Require `plugin_dbui` 0.7.3

See merge request !49

.gitignore

@@ -23,6 +23,6 @@ static/plugin_ace
 static/plugin_dbui
 static/plugin_extjs
 static/plugin_mathjax
-static/track-publication*.js
+static/track_publications*.js
 views/plugin_dbui
 controllers/plugin_dbui.py

build_version.py

@@ -36,6 +36,7 @@ from subprocess import call
 APP = os.path.basename(os.getcwd())
 API = 'api'
 CHANGELOG = 'static/CHANGELOG'
+DBUISRC = 'static/plugin_dbui/src'
 DOCS = 'static/docs'
 DOCSRC = 'docs'
 EXTJSSRC = 'static/plugin_extjs/src'
@@ -202,22 +203,20 @@ def compile_js():
             print 'Remove old javascript library', item
 
     # debug version of the javascript library
-    cwd = os.getcwd()
-
-    cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
-           "compile", "-class", opj(cwd, JSLIBSRC),
-           "exclude", "--namespace", "Ext",
-           "and", "concat", opj(cwd, JSLIBDEBUG)]
+    cmd = ["sencha", "-sdk", EXTJSSRC,
+           "compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
+           "exclude", "--namespace", "Ext,App",
+           "and", "concat", JSLIBDEBUG]
 
     call(cmd)
 
     print 'Debug version of the javascript library', JSLIBDEBUG, 'is ready'
 
     # Minified version of the javascript library
-    cmd = ["sencha", "-sdk", opj(cwd, EXTJSSRC),
-           "compile", "-class", opj(cwd, JSLIBSRC),
-           "exclude", "--namespace", "Ext",
-           "and", "concat", "--yui", opj(cwd, JSLIBMIN)]
+    cmd = ["sencha", "-sdk", EXTJSSRC,
+           "compile", "-classpath", "%s,%s" % (JSLIBSRC, DBUISRC),
+           "exclude", "--namespace", "Ext,App",
+           "and", "concat", "--yui", JSLIBMIN]
 
     call(cmd)
 
@@ -419,7 +418,9 @@ if __name__ == '__main__':
 
     APS.add_argument("-C", "--commit-changelog",
                      action="store_true",
-                     help="commit CHANGELOG.")
+                     help="commit CHANGELOG. "
+                          "To be used with --write-release. "
+                          "Recommend to use --start-release.")
 
     APS.add_argument("-j", "--jsduck",
                      action="store_true",
@@ -435,7 +436,11 @@ if __name__ == '__main__':
 
     APS.add_argument("-s", "--start-release",
                      action="store_true",
-                     help="start the new release.")
+                     help="start the new release cycle. "
+                          "Set the release number in the changelog and "
+                           "commit changes in the master branch. "
+                           "Equivalent to --write-release followed by "
+                           "--commit-changelog.")
 
     APS.add_argument("-u", "--user-doc",
                      action="store_true",
@@ -451,7 +456,9 @@ if __name__ == '__main__':
 
     APS.add_argument("-w", "--write-release",
                      action="store_true",
-                     help="write the release number in CHANGELOG.")
+                     help="write the release number in CHANGELOG. "
+                          "To be used with --commit-changelog. "
+                          "Recommend to use --start-release.")
 
     ARGS = APS.parse_args()
 

controllers/harvest.py

@@ -20,7 +20,7 @@ from invenio_tools import (load_record,
                            RecordThesis,
                            REG_INT)
 from plugin_dbui import (get_id,
-                         INLINE_ALERT,
+                         inline_alert,
                          Selector,
                          to_formPanel,
                          UNDEF_ID)
@@ -36,7 +36,7 @@ def free_run():
 
     """
     if not current.app.inspirehep_institute_id:
-        return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
+        return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
 
     table = virtdb.free_harvester_selector
     fields = ('collections',
@@ -56,7 +56,7 @@ def free_run():
                 msg = T('All fields of the form have to be defined !!!')
                 msg += "<br>"
                 msg += T('The field "%s" is missing ...') % T(table[el].label)
-                return INLINE_ALERT % (T('Error'), msg)
+                return inline_alert(T('Error'), msg)
 
         tool = build_harvester_tool(db,
                           selector.id_teams,
@@ -68,7 +68,7 @@ def free_run():
                           dry_run=(selector.mode == MODE_DRY_RUN),
                           debug=False)
         if not tool:
-            return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
+            return inline_alert(T('Error'), T('Select an harvester.'))
 
         tool.process_url(selector.host, selector.collections)
 
@@ -96,7 +96,7 @@ def edit_insert():
 
     """
     if not current.app.inspirehep_institute_id:
-        return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
+        return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
 
     fields = ('controller',
               'host',
@@ -118,7 +118,7 @@ def edit_insert():
             msg = T("The <i>record id</i> is not well formed.")
             msg += "<br>"
             msg += T("Use only digit character, no comma, no dot...")
-            return INLINE_ALERT % (T('Error'), msg)
+            return inline_alert(T('Error'), msg)
 
         selector = Selector(table)
 
@@ -127,17 +127,13 @@ def edit_insert():
                 msg = T('All fields of the form have to be defined !!!')
                 msg += "<br>"
                 msg += T('The field "%s" is missing ...') % T(table[el].label)
-                return INLINE_ALERT % (T('Error'), msg)
+                return inline_alert(T('Error'), msg)
 
         # record
         record = load_record(selector.host, selector.record_id)
 
         if record is None:
-            # NOTE
-            # Bug in plugin dbui 0.7.1 in INLINE_ALERT
-            msg = '<script>Ext.Msg.alert("%s", "%s");</script>'
-            msg = msg % (T('Error'), T(MSG_NO_RECORD))
-            return msg
+            return inline_alert(T('Error'), T(MSG_NO_RECORD))
 
         # form configuration
         cfg = to_formPanel(db.publications)
@@ -283,7 +279,7 @@ def insert_marcxml():
 
     """
     if not current.app.inspirehep_institute_id:
-        return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
+        return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
 
     try:
         selector = Selector(virtdb.marc12_selector, exclude_fields=('mode'))
@@ -298,7 +294,7 @@ def insert_marcxml():
                                     dry_run=(selector.mode == MODE_DRY_RUN),
                                     debug=False)
         if not tool:
-            return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
+            return inline_alert(T('Error'), T('Select an harvester.'))
 
         tool.process_xml(selector.xml)
 
@@ -331,7 +327,7 @@ def run():
 
     """
     if not current.app.inspirehep_institute_id:
-        return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
+        return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
 
     try:
         selector = Selector(virtdb.harvester_selector,
@@ -356,7 +352,7 @@ def run():
                                         dry_run=(selector.mode == MODE_DRY_RUN),
                                         debug=False)
             if not tool:
-                return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
+                return inline_alert(T('Error'), T('Select an harvester.'))
 
             tool.process_url(row.harvesters.host, row.harvesters.collections)
 
@@ -385,7 +381,7 @@ def run_all():
 
     """
     if not current.app.inspirehep_institute_id:
-        return INLINE_ALERT % (T("Error"), T(MSG_NO_REG_INSTITUTE))
+        return inline_alert(T("Error"), T(MSG_NO_REG_INSTITUTE))
 
     collection_logs = []
     logs = []
@@ -405,7 +401,7 @@ def run_all():
 
         harvesters = db(query).select(db.harvesters.ALL)
         if not len(harvesters):
-            return INLINE_ALERT % (T('Error'), T(MSG_NO_HARVESTER))
+            return inline_alert(T('Error'), T(MSG_NO_HARVESTER))
 
         for harvester in harvesters:
 
@@ -419,7 +415,7 @@ def run_all():
                                         dry_run=(selector.mode == MODE_DRY_RUN),
                                         debug=False)
             if not tool:
-                return INLINE_ALERT % (T('Error'), T('Select an harvester.'))
+                return inline_alert(T('Error'), T('Select an harvester.'))
 
             tool.process_url(harvester.host, harvester.collections)
 

controllers/metrics.py

@@ -5,12 +5,10 @@ import json
 import re
 
 from gluon.dal import smart_query
-from plugin_dbui import Selector
+from plugin_dbui import inline_alert, Selector
 from reporting_tools import get_converter, repr_team_project
 
-INLINE_ALERT = "<script>Ext.Msg.alert('%s', '%s');</script>"
-MSG_NO_METRIC = T("Please select a metric....")
-MSG_NO_TABLE = T("Invalid database table '%s'")
+MSG_NO_METRIC = "Please select a metric...."
 
 
 def index():
@@ -27,7 +25,7 @@ def index():
 
     # protection
     if not selector.id_metrics:
-        return INLINE_ALERT % (T("Error"), MSG_NO_METRIC)
+        return inline_alert(T("Error"), T(MSG_NO_METRIC))
 
     # retrieve metric data
     metric = db.metrics[selector.id_metrics]

controllers/wizards.py

@@ -8,18 +8,18 @@ import re
 from check_tools import check_publication
 from gluon.storage import Storage
 from harvest_tools import DRY_RUN
-from plugin_dbui import (is_foreign_field,
+from plugin_dbui import (CALLBACK_ERRORS,
+                         inline_alert,
+                         is_foreign_field,
                          get_foreign_field,
+                         get_id,
                          Selector,
                          to_fields)
 
 
 MODE_DRY_RUN = T(DRY_RUN)
-
-INLINE_ALERT = "<script>Ext.Msg.alert('%s', '%s');</script>"
-
 MSG_NO_AUTHORS = "<br><br>Removing affiliation failed.<br>"\
-"Use INSPIRES instead with the tool 'insert MARCXML'"
+    "Use INSPIRES instead with the tool 'insert MARCXML'"
 
 
 def check_validate():
@@ -28,7 +28,7 @@ def check_validate():
     """
     counters = {}
     logs = []
-    id_ok = db(db.status.code=='OK').select().first().id
+    id_ok = db(db.status.code == 'OK').select().first().id
 
     # get user requirement
     selector = Selector(virtdb.check_selector, exclude_fields=('mode'))
@@ -89,7 +89,7 @@ def compare_publications():
     data, idrow = [], []
 
     if 'id1' not in request.vars or 'id2' not in request.vars:
-        return INLINE_ALERT % (T('Error'), T('Specify id1 and id2 in the URL'))
+        return inline_alert(T('Error'), T('Specify id1 and id2 in the URL'))
 
     row1 = db.publications[request.vars.id1]
     row2 = db.publications[request.vars.id2]
@@ -188,4 +188,116 @@ def extract_authors():
     if not case_1:
         return MSG_NO_AUTHORS
 
-    return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
\ No newline at end of file
+    return dict(all=', '.join(all_authors), my_authors=', '.join(my_authors))
+
+
+def harvester():
+    """Process the data send by the wizard harvester and fill the
+    harvesters table. The data block contains the keys:
+
+        * automaton (str)
+        * collaboration (str)
+        * collection (str)
+        * people (list)
+        * producer (str) either 'collaboration' or 'people'
+        * project (int)
+        * store (str) either cds.cern.ch or inspirehep.net
+
+    The logic to interpret the data depends on the store.
+
+    """
+    # shortcuts
+    vars = request.vars
+    automaton = vars.automaton
+    organisation = db.organisation
+    project = vars.project
+    store = vars.store
+
+    # protection
+    if store not in ("cds.cern.ch", "inspirehep.net"):
+        raise HTTP(500, "Invalid store !")
+
+    values = Storage(controller=automaton,
+                     host=store,
+                     id_projects=project)
+
+    # the team
+    rec_id = get_id(organisation, id_projects=project)
+    if rec_id is None:
+        raise HTTP(500, "Project is unknown !")
+
+    values.id_teams = organisation[rec_id].id_teams
+
+    # the collection
+    if store == "cds.cern.ch":
+        values.collections = vars.collection
+
+    elif store == "inspirehep.net":
+        collection = []
+
+        collaboration = vars.collaboration
+        if collaboration:
+            collection.append("cn %s" % collaboration)
+
+        else:
+            collection.append("(a %s)" % " or ".join(vars.people))
+
+        if automaton == "articles":
+            collection.append("tp p and not tc c")
+
+        elif automaton == "proceedings":
+            collection.append("tp p and tc c")
+
+        values.collections = "find %s" % " and ".join(collection)
+
+    # the publication category
+    if automaton == "articles":
+        id_category = get_id(db.categories, code="ACL")
+
+    elif automaton in ("notes", "reports"):
+        id_category = get_id(db.categories, code="AP")
+
+    elif automaton == "preprints":
+        id_category = get_id(db.categories, code="PRE")
+
+    elif automaton == "proceedings":
+        id_category = get_id(db.categories, code="ACTI")
+
+    elif automaton == "talks":
+        id_category = get_id(db.categories, code="COM")
+
+    elif automaton == "theses":
+        id_category = get_id(db.categories, code="PHD")
+
+    if id_category is None:
+        raise HTTP(500, "Category is unknown !")
+
+    values.id_categories = id_category
+
+    # insert new values in the database
+    msg = None
+    try:
+        rec_id = db.harvesters.insert(**values)
+        if rec_id:
+            return
+
+        # operation can be reject by callback table._before_insert
+        else:
+            msg = "Fail to insert the new harvester in the database."
+            if CALLBACK_ERRORS in db.harvesters:
+                msg = db.harvesters._callback_errors
+
+            # reduce the error message
+            if isinstance(msg, list):
+                msg = "%s %s" % (msg[0], msg[-1])
+
+    # operation is rejected by the database
+    except Exception as dbe:
+        raise HTTP(500, dbe.message)
+
+    # operation is rejected by the callback
+    # NOTE in the else branch to avoid recursive exception generation
+    if msg is not None:
+        raise HTTP(500, msg)
+
+    return

docs/api/generated/callbacks.INHIBIT_HARVESTER.rst

+callbacks.INHIBIT_HARVESTER
+============================
+
+.. currentmodule:: callbacks
+
+.. autofunction:: INHIBIT_HARVESTER
\ No newline at end of file

docs/api/generated/callbacks.INHIBIT_HARVESTER_ON_CATEGORY.rst

-callbacks.INHIBIT_HARVESTER_ON_CATEGORY
-=======================================
-
-.. currentmodule:: callbacks
-
-.. autofunction:: INHIBIT_HARVESTER_ON_CATEGORY
\ No newline at end of file

docs/api/tools.rst

@@ -23,7 +23,7 @@ the-database-abstraction-layer#callbacks-on-record-insert--delete-and-update>`_.
 
         ~INHIBIT_CASCADE_DELETE
         ~INHIBIT_DUPLICATE_PUBLICATION
-        ~INHIBIT_HARVESTER_ON_CATEGORY
+        ~INHIBIT_HARVESTER
         ~INHIBIT_PUBLICATION_DELETE_ON_OK
         ~INHIBIT_PUBLICATION_UPDATE_ON_OK
 

languages/fr-fr.py

@@ -88,6 +88,7 @@
 "Can't delete this record since several publications refer to it.": 'Impossible de détruire cet enregistrement car des publications lui font référence.',
 "Can't insert the article.": 'Impossible de créer cet article.',
 "Can't insert the harvester.": 'Impossible de créer ce moissonneur.',
+"Can't insert the harvester. ": "Impossible de créer ce moissonneur. ",
 "Can't insert the report.": 'Impossible de créer ce rapport.',
 "Can't insert the talk/proceeding.": 'Impossible de créer cet acte ou présentation.',
 "Can't updated a publication marked OK.": "Impossible d'actualiser une publication marquée OK.",
@@ -293,6 +294,7 @@
 'HTTP Error': 'HTTP Error',
 'id': 'id',
 'Id': 'Id',
+'Identical harvester already exists.': 'Ce moissonneur existe dans la base de données.',
 'Import/Export': 'Importer/Exporter',
 'insert MARCXML': 'insérer MARCXML',
 'insert new': 'insert new',

models/common_settings.py

@@ -12,7 +12,7 @@ from callbacks import (INHIBIT_CASCADE_DELETE,
                        INHIBIT_CONTROLLER_INSERT,
                        INHIBIT_CONTROLLER_UPDATE,
                        INHIBIT_DUPLICATE_PUBLICATION,
-                       INHIBIT_HARVESTER_ON_CATEGORY,
+                       INHIBIT_HARVESTER,
                        INHIBIT_PUBLICATION_DELETE_ON_OK,
                        INHIBIT_PUBLICATION_UPDATE_ON_OK)
 
@@ -31,8 +31,9 @@ from regex import (REG_COLLABORATION,
 #-------------------------------------------------------------------------------
 plugins = PluginManager()
 plugins.dbui.app_css = 'static/my.css'
-# plugins.dbui.app_js_dir = None
-# plugins.dbui.app_libmin = None
+plugins.dbui.app_debug = None
+plugins.dbui.app_lg = 'static/track_publications/locale/trp-lang-fr.js'
+plugins.dbui.app_libmin = 'static/track_publications-min.js'
 plugins.dbui.app_script = 'static/app.js'
 # plugins.dbui.app_script_dir = None
 

models/db1_harvesters.py

@@ -24,7 +24,7 @@ db.define_table("harvesters",
     Field("scan", "boolean", comment=tp_scan, default=False),
     migrate="harvesters.table")
 
-db.harvesters._before_insert.append(INHIBIT_HARVESTER_ON_CATEGORY)
+db.harvesters._before_insert.append(INHIBIT_HARVESTER)
 
 db.harvesters.collections.filter_in = dbui.CLEAN_SPACES
 db.harvesters.controller.filter_in = dbui.CLEAN_SPACES

models/ui_harvesters.py

@@ -54,6 +54,9 @@ gridModifier.append_filter(('id_categories', '==', T('is equal to')))
 gridModifier.configure_filters(plugins=['pFormToolTip'], width=300)
 gridModifier.configure_gridWithFilter(selectorTitle=T('Filter'))
 
+gridModifier.configure(plugins=[{'ptype': 'pGridRowEditorAddWizard',
+                                 'wizard' :{'xtype': 'xwizardharvester'}}])
+
 #-------------------------------------------------------------------------------
 #
 # STORE CONFIGURATiON

modules/callbacks.py

@@ -187,12 +187,15 @@ def INHIBIT_DUPLICATE_PUBLICATION(publication):
     return False
 
 
-def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
-    """Inhibit the insert of similar harvesters.
+def INHIBIT_HARVESTER(harvester):
+    """Inhibit the insert of similar harvesters:
 
-    For a project, one automaton can only proceed publication
-    of the same code, e.g ACL or ACLN but not both, but automatons
-    can scan different stores.
+        * For a project, one automaton can only proceed publication
+          of the same code, e.g ACL or ACLN but not both, but automatons
+          can scan different stores.
+
+        * Reject identical harvester, same project, controller, store,
+          collection and category.
 
     Args:
         harvester (dict): harvester fields passed to insert.
@@ -209,7 +212,7 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
                           id_projects=harvester['id_projects'],
                           id_teams=harvester['id_teams'])
 
-    if not id_harvester:
+    if id_harvester is None:
         return False
 
     # for a project, one automaton can only proceed publication
@@ -226,6 +229,23 @@ def INHIBIT_HARVESTER_ON_CATEGORY(harvester):
 
         return True
 
+    # duplicate harvesters
+    id_harvester = get_id(db.harvesters,
+                          collections=harvester['collections'],
+                          controller=harvester['controller'],
+                          host=harvester['host'],
+                          id_categories=harvester['id_categories'],
+                          id_projects=harvester['id_projects'],
+                          id_teams=harvester['id_teams'])
+
+    if id_harvester is not None:
+
+        db.harvesters[CALLBACK_ERRORS] = [
+            T("Can't insert the harvester. "),
+            T("Identical harvester already exists.")]
+
+        return True
+
     return False
 
 

modules/harvest_tools/base.py

@@ -182,7 +182,7 @@ def search_synonym(table, fieldname, value, create=False):
         return id_rec
 
     # nothing found, have a look to the synonyms field
-    query = table.synonyms.contains(value)
+    query = table.synonyms.contains(value.encode('utf-8'))
     setrows = db(query)
 
     # no synonym found, create the entry

static/ABOUT.html

@@ -60,10 +60,20 @@
 		        MathJax
 	        </a>
         </li>
+        <li class="my-li">
+            <a href="http://matplotlib.org/" target="_blank">
+                Matplotlib
+            </a>
+        </li>
         <li class="my-li">
             <a href="https://marprod.in2p3.fr/plugin_dbui_book" target="_blank">
                 plugin dbui
             </a>
+        </li>
+        <li class="my-li">
+            <a href="http://pandas.pydata.org/" target="_blank">
+                Pandas
+            </a>
         </li>
 		<li class="my-li">
 		    <a href="http://www.python.org/" target="_blank">
@@ -88,6 +98,6 @@
     <p class="my-p">Ce logiciel est distribué sous la licence opence source
     <a href="http://www.cecill.info/" target="_blank">CeCILL</a>.</p>
 
-	<p class="my-p">&copy; 2010 - 2014 Renaud Le Gac</p>
+	<p class="my-p">&copy; 2010 - 2015 Renaud Le Gac</p>
 </body>
 </html>
\ No newline at end of file

static/CHANGELOG

@@ -2,13 +2,17 @@
 
 HEAD
 
+0.9.3 (Dec 2015)
+  - Require plugin_dbui 0.7.3
+  - Add the wizard to create an harvester.
+  
 0.9.2 (Nov 2015)
   - Require plugin_dbui 0.7.2 or later release.
   - Major review of the user guide.
   - Protect the table controllers against duplicate pair (harvester, category)
     as well as duplicate usage of the publication category.
   - Several bugs fixed.
-  - Fix bug in the list extraction to CSF file.
+  - Fix bug in the list extraction to CSV file.
   - Fix a bug in the graph generation when the database is empty.
   
 0.9.0 (Nov 2015)

static/app.js

@@ -10,18 +10,23 @@ if (App.debug) {
         enabled: true,
         paths: {
             'App': '/' + App.name + '/static/plugin_dbui/src',
-            'Ext': '/' + App.name + '/static/plugin_extjs/src'
+            'Ext': '/' + App.name + '/static/plugin_extjs/src',
+            'Trp': '/' + App.name + '/static/track_publications/src'
         }
     });
 }
 
 Ext.require('App.container.Viewport');
+Ext.require('App.grid.plugin.RowEditorAddWizard');
 Ext.require('App.plugin.MathJax');
+
 Ext.require('Ext.direct.Manager');
 Ext.require('Ext.direct.RemotingProvider');
 Ext.require('Ext.EventManager');
 Ext.require('Ext.tip.QuickTipManager');
 
+Ext.require('Trp.wizard.Harvester');
+
 Ext.onReady(function(){
 
     "use strict";

static/track_publications/locale/trp-lang-en.js

+/**
+ * Mother language