Do no reject a record with a bad OAI, recover it.

modules/harvest_tools/automaton.py

@@ -6,7 +6,7 @@ import re
 import traceback
 
 
-from base import MSG_FIX_ORIGIN, MSG_IN_DB, ToolException
+from base import MSG_FIX_ORIGIN, MSG_IN_DB, recover_oai, ToolException
 from gluon.storage import Storage
 from invenio_tools import (CheckAndFix,
                            InvenioStore,
@@ -392,17 +392,17 @@ class Automaton(object):
         """Decode the xml and load it in the database.
 
         @raise Exception: the type of exception depends on what happen:
-            - L{ToolException} when projet, team or category identifier
+            - L{ToolException} when project, team or category identifier
               are not defined.
-            - C{StoreException} when somethings goes wrong interrogating the
-                                store.
-            - C{Marc12Exception} when somethings goes wrong decoding the XML
+            - C{StoreException} when something goes wrong interrogating the
+              store.
+            - C{Marc12Exception} when something goes wrong decoding the XML
               string return by the store
             - C{CheckException} if the L{Record} is not valid
             - C{Exception} if the python code crash
 
         @type xml: unicode
-        @keyword xml: marc12 xml encoding of the publication record
+        @keyword xml: marc12 XML encoding of the publication record
 
         """
         if self.dbg:
@@ -417,7 +417,7 @@ class Automaton(object):
         self.decode_xml(xml)
 
     def process_url(self, host, collections):
-        """Retrieve the xml from the invenio store and load it in the database
+        """Retrieve the XML from the invenio store and load it in the database
 
         @raise Exception: depending on what happen, can be StoreException,
         Marc12ZException, ...
@@ -528,17 +528,14 @@ class Automaton(object):
                                  record_id=record.id(),
                                  title=record.title()))
 
-            # reject record with undefined OAI field
+            # the OAI is not defined -- recover it
             oai = record.oai()
             if oai is None:
-                self.logs[-1].reject(MSG_NO_OAI, record.year())
-                continue
+                recover_oai(record, self.harvester.host)
 
-            # reject the record when the OAI is not well
-            match = REG_OAI.match(oai)
-            if not match:
-                self.logs[-1].reject(MSG_WELL_FORM_OAI, record.year())
-                continue
+            # the OAI is not well --recover it
+            if not REG_OAI.match(oai):
+                recover_oai(record, self.harvester.host)
 
             # check that the record is well formed
             # repair non-conformity as far as possible

modules/harvest_tools/base.py

@@ -14,6 +14,10 @@ MSG_FIX_ORIGIN = "Fixed the origin field"
 MSG_IN_DB = "Already in the database"
 MSG_LOAD = "Load in the database"
 
+MSG_INVALID_HOST = "Invalid host"
+
+OAI_INVENIO = "oai:%s:%s"
+
 
 def family_name_fr(full_name):
     """Extract the family name when the full name is encoded as C{J. Doe}.
@@ -141,5 +145,28 @@ def learn_my_authors(db,
         db.my_authors[row.id] = dict(authors=', '.join(database_authors))
 
 
+def recover_oai(record, host):
+    """Helper function to recover the OAI identifier when it is not defined
+    or not well form.
+
+    @type record: Record
+    @param record:
+
+    """
+    if host == "cds.cern.ch":
+        field, subfield = u"0248", "a"
+
+    elif host == "inspirehep.net":
+        field, subfield = u"909CO", "o"
+
+    else:
+        raise ValueError(MSG_INVALID_HOST)
+
+    if field not in record:
+        record[field] = dict()
+
+    record[field][subfield] = OAI_INVENIO % (host, record.id())
+
+
 class ToolException(Exception): pass
 

modules/invenio_tools/checkandfix.py

@@ -21,18 +21,19 @@ from recordconf import RecordConf
 from recordthesis import RecordThesis
 
 
-DECODE_ARXIV = re.compile("arXiv:(\d{2})(\d{2})\.")
+DECODE_ARXIV = re.compile(r"arXiv:(\d{2})(\d{2})\.")
 
 # Decode submitted date: DD MMM YYYY or DD MM YYY
 DECODE_DD_MMM_YYYY = re.compile(r"(\d{1,2}) *([A-Za-z]{3}) *(\d{4})")
 DECODE_DD_MM_YYYY = re.compile(r"(\d{1,2}) +(\d{1,2}) +(\d{4})")
-DECODE_YYYY = re.compile("^(\d{4})$")
+DECODE_YYYY = re.compile(r"^(\d{4})$")
 
 # Decode publication reference:
 #  Phys. Rev. Lett. 113, 032001 (2014)
 #  Eur. Phys. J. C (2014) 74:2883
-DECODE_REF = [re.compile("(?P<p>[A-Za-z\. ]+) +(?P<v>\d+),? +(?P<c>[\d-]+) +\((?P<y>[\d]+)\)"),
-              re.compile("(?P<p>[A-Za-z\. ]+) +\((?P<y>\d+)\) +(?P<v>[\d]+):(?P<c>[\d-]+)")]
+_ref1 = r"(?P<p>[A-Za-z\. ]+) +(?P<v>\d+),? +(?P<c>[\d-]+) +\((?P<y>[\d]+)\)"
+_ref2 = r"(?P<p>[A-Za-z\. ]+) +\((?P<y>\d+)\) +(?P<v>[\d]+):(?P<c>[\d-]+)"
+DECODE_REF = [re.compile(_ref1), re.compile(_ref2)]
 
 MONTHS = {u'Jan':'01',
           u'Feb':'02',
@@ -70,11 +71,12 @@ MSG_WELL_FORMED_DATE = "Reject submission date is not well formed"
 
 MSG_WELL_FORMED_EDITOR = "Reject editor is not well formed"
 
-OAI_INVENIO = "oai:%s:%s"
-
 REG_COLLABORATION = re.compile(regex.REG_COLLABORATION)
 REG_CONF_DATES_1 = re.compile("(\d+) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})")
-REG_CONF_DATES_2 = re.compile("(\d+) *([A-Z][a-z]{2}) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})")
+
+REG_CONF_DATES_2 = \
+re.compile("(\d+) *([A-Z][a-z]{2}) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})")
+
 REG_CONF_DATES = re.compile(regex.REG_CONF_DATES)
 REG_SUBMITTED = re.compile(regex.REG_SUBMITTED)
 
@@ -483,7 +485,7 @@ class CheckAndFix(object):
             - INVENIO:    Phys. Lett. B + volume 673
             - INSPIREHEP: Phys.Lett + volume B673
 
-        Standardise the answer as Phys Lett B
+        Standardize the answer as Phys Lett B
 
         @note: It is recommended to call this method when erratum are removed.
 
@@ -595,7 +597,8 @@ class CheckAndFix(object):
 
     def get_my_authors(self, record, cmpFct=None):
         """Get authors of my institutes signing the record.
-        The information is append to the L{Record} via the attribute C{my_authors}.
+        The information is append to the L{Record} via the attribute
+        C{my_authors}.
 
         @type record: L{Record}
         @param record:
@@ -764,13 +767,15 @@ class CheckAndFix(object):
             # 22 Mar 2011
             m = DECODE_DD_MMM_YYYY.match(dates[i])
             if m:
-                dates[i] = '%s-%s-%02i' % (m.group(3), MONTHS[m.group(2)], int(m.group(1)))
+                data = (m.group(3), MONTHS[m.group(2)], int(m.group(1)))
+                dates[i] = '%s-%s-%02i' % data
                 continue
 
             # 22 03 2011
             m = DECODE_DD_MM_YYYY.match(dates[i])
             if m:
-                dates[i] = '%s-%02i-%02i' % (m.group(3), int(m.group(2)), int(m.group(1)))
+                data (m.group(3), int(m.group(2)), int(m.group(1)))
+                dates[i] = '%s-%02i-%02i' % data
                 continue
 
             # 2011