automaton.py

# -*- coding: utf-8 -*-
""" harvest_tools.automaton

"""
import re
import traceback


from base import (MSG_FIX_ORIGIN,
                  MSG_IN_DB,
                  search_synonym,
                  ToolException)
from checkandfix import CheckAndFix
from gluon.storage import Storage
from invenio_tools import (InvenioStore,
                           Marc12,
                           OAI_URL)
from msg import Msg
from msgcollection import MsgCollection
from plugin_dbui import CALLBACK_ERRORS, get_id


MSG_NO_CAT = 'Select a "category" !!!'
MSG_NO_PROJECT = 'Select a "project" !!!'
MSG_NO_TEAM = 'Select a "team" !!!'

MSG_NSERT_FAIL = "Fail to insert the new record in the database."
MSG_NO_OAI = "Reject no OAI identifier"
MSG_WELL_FORM_OAI = "Reject OAI is not well formed"

# search collection when using inspirehep
# require for "Hal Hidden"
REG_COLLECTION = re.compile(r"cc([A-Za-z ]+)(and|$)")


class Automaton(object):
    """Base class to search and process publications:

        * Decode the selector defining user criteria.
        * Search in the store publications matching user criteria.
        * Decode the XML string return by the store.
        * Insert new records in the database.

    Note:
        The parameters of the search are defined by the current ``request``.

    The logic implements in the ``Automaton`` class is the following:

        #. Ask to the store, all the `record_id` satisfying the user request.
        #. Reject `record_id` contains in the *origin* field of a
           database entry.
        #. Request to the store, the XML description of the publications
           and decode them.
        #. Reject the record for which the *secondary_oai_url* is contained in
           the *origin* field of a database entry. Update the *origin* field
           of the database record.
        #. Check that the *oai* of the publication is defined and well formed.
           Recover it, if it is not the case. At this stage the OAI is always
           defined.
        #. Reject temporarily publication.
        #. Check that *authors* are defined.
           Reject the publication if it is not the case.
        #. Check that *my institute* is in the list of the institutes
           signing the publication. Reject the publication if it is
           not the case. When the affiliation are not defined,
           try to recover this case, by finding the author of my institute
           signing the publication. This recovery procedure uses
           the *author rescue list*. Reject the record when the recovery
           procedure failed.
        #. Check that the *collaboration*, if defined, is well formed.
           Reject the publication if it is not the case
        #. Several check are applied depending on the publication type.
        #. At the end of this process, the publisher, the authors are
           formatted and the list of signatories of my institute extracted.

    Args:
        db (gluon.DAL): the database connection.
        id_team (int): the identifier of the team in the database.
        id_project (int): the identifier of the project in the database.
        automaton (unicode): the name of the automaton which
            will be used to process the data. Possible values are:
            ``articles``, ``notes``, ``preprints``, ``proceedings``,
            ``reports``, ``talks`` and ``theses``.
        id_category (int): the identifier of the category of publication
        year_start (int): starting year for the scan
        year_end (int): ending year of the scan
        dry_run (bool): new records are not inserted in the database
            when ``True``.
        debug (bool): activate the verbose mode when ``True``.

    Raises:
        ToolException: when the team or the project or the publication category
            is not defined.

    """
    def __init__(self,
                 db,
                 id_team,
                 id_project,
                 automaton,
                 id_category,
                 year_start=None,
                 year_end=None,
                 dry_run=True,
                 debug=False):

        self.collection_logs = []
        self.db = db
        self.id_team = id_team
        self.id_project = id_project
        self.controller = automaton
        self.id_category = id_category
        self.year_start = year_start
        self.year_end = year_end
        self.dry_run = dry_run
        self.dbg = debug
        self.logs = []

        self.check = CheckAndFix()
        self.marc12 = Marc12()

        # check parameters
        # protection team, project and/or category have to be defined
        if not self.id_team:
            raise ToolException(MSG_NO_TEAM)

        if not self.id_project:
            raise ToolException(MSG_NO_PROJECT)

        if not self.id_category:
            raise ToolException(MSG_NO_CAT)

        # Construct harvester Storage needed for the log
        self.harvester = Storage(id_teams=self.id_team,
                                 id_projects=self.id_project,
                                 controller=self.controller,
                                 id_categories=self.id_category)

        # Identifier of the categories preprint and articles
        # Used by the method _is_record_in_db
        self._id_preprint = get_id(db.categories, code="PRE")
        self._id_article = get_id(db.categories, code="ACL")

    def _insert_in_db(self, log_year="", **fields):
        """Insert the record in the database, handling database exception.

        Args:
            log_year (unicode): year of the record for the log

        Keyword Args:
            **fields: keyword arguments defining the record values to be
                inserted in the database.

        Returns:
            int: one when the record is inserted / updated in the database,
                zero otherwise.

        """
        db = self.db

        try:
            rec_id = db.publications.insert(**fields)
            if rec_id:
                return 1

            # operation can be reject by callbacks table._before_insert
            else:
                msg = MSG_NSERT_FAIL
                if CALLBACK_ERRORS in db.publications:
                    msg = db.publications._callback_errors

                # reduce the error message
                if isinstance(msg, list):
                    msg = "%s %s" % (msg[0], msg[-1])

                self.logs[-1].reject(msg, log_year)
                return 0

        # operation can be rejected by the database
        except Exception as dbe:
            self.logs[-1].reject(dbe.message, log_year)
            return 0

    def _is_record_in_db(self, title, host=None, rec_id=None, oai_url=None):
        """Return ``True`` when the record is already in the database.
        The search is based on the origin field and on the primary OAI.

        Note:
            A new log entry is created when a record is found.

        Args:
            title (unicode): the title of the publication.

        Keyword Args:
            host (unicode): the store. possible values are ``cds.cern.ch`` or
                ``inspirehep.net``. To be used with *rec_id*.
            rec_id (int): the record identifier in the store
            oai_url (unicode): the URL of the record in the store.

                Note:
                    Either use *host* and *rec_id* or *oai_url*

        Returns:
            int: the id of the record in the database when a record is found,
                0 otherwise.

        Raises:
            ValueError: when keyword arguments are not defined properly.

        """
        db = self.db
        harvester = self.harvester

        # build the OAI URL
        if host is not None and rec_id is not None and oai_url is None:
            url = OAI_URL % (host, rec_id)
        elif host is None and rec_id is None and oai_url is not None:
            url = oai_url
        else:
            raise ValueError

        # protection empty url
        if len(url) == 0:
            return 0

        # check the OAI
        query = db.publications.origin.contains(url)
        setrows = db(query)

        if setrows.count() == 0:
            return 0

        # one record found
        columns = [db.publications.id,
                   db.publications.id_categories,
                   db.publications.title,
                   db.publications.year]
        publication = setrows.select(*columns).first()

        # Note:
        # The category for the publication and the harvester have to be equal.
        # However, keep the record if it is a preprint when the harvester
        # looks for articles. This is required to transform a preprint
        # into article
        #
        # Category can disagree when the publication is an article and
        # the harvester look for preprint. In that case, keep the article
        #
        if publication.id_categories != harvester.id_categories:

            is_preprint_to_article = \
                publication.id_categories == self._id_preprint \
                and harvester.id_categories == self._id_article

            if is_preprint_to_article:
                return 0

        # log
        self.logs.append(Msg(harvester=harvester,
                             collection=title,
                             record_id=rec_id,
                             title=publication.title))

        self.logs[-1].idle(MSG_IN_DB, publication.year)

        return publication.id

    def _search_parameters(self, collection):
        """Build the keywords to steer the URL search in invenio store.
        The main parameter is the collection and the date range defined
        in the selector.

        Args:
            collection (unicode): string defining the collection in the
                store. The syntax depends on the invenio store:

                    * ``"find cn d0 and tc p and not tc c"``
                    * ``"LHCb Papers"``.

        Returns:
            dict: the key are a sub-set of those defined in
                :py:meth:invenio_tools.InvenioStore.get_ids.

        """

        # INSPIREHEP store
        if collection.startswith('find'):

            query = collection

            if self.year_start and not self.year_end:
                query += " and date %s" % self.year_start

            elif not self.year_start and self.year_end:
                query += " and date %s" % self.year_end

            elif self.year_start and self.year_end:
                query += " and date > %s and date < %s " \
                         % (self.year_start - 1, self.year_end + 1)

            dic = dict(p=query,  # query à la spires
                       rg=1000,  # maximum number of records returned
                       sf='year',  # sort by date
                       so='d')  # descending order

            # handle the cc keyword (true inspirehep collection)
            match = REG_COLLECTION.search(query)
            if match:
                dic["cc"] = match.group(1).strip()
                dic["p"] = REG_COLLECTION.sub("", query).strip()
                dic["p"] = dic["p"].replace("  ", " ")
                if dic["p"] == "find":
                    del dic["p"]

        # CERN INVENIO store
        else:

            if self.year_start and not self.year_end:
                rex = self.year_start

            elif not self.year_start and self.year_end:
                rex = self.year_end

            elif self.year_start and self.year_end:
                li = []
                for year in range(self.year_start, self.year_end + 1):
                    li.append(str(year))
                rex = '|'.join(li)

            dic = dict(cc=collection,  # collection
                       f1='year',  # search on year
                       m1='r',  # use regular expression
                       p1=rex,  # regular expression defining year
                       sf='year',  # sort by date
                       so='d')  # descending order
        return dic

    def check_record(self, record):
        """Check the content of the record in order to fix non-conformities.
        Return ``False`` when non-conformities are found and can not be
        corrected.

        Note:
            Some checks depend on the type of publications and have to be
            implemented in inherited class.

        Note:
            The order of the checks matter. It should be oai,
            temporary record, authors, my authors and then a series of checks
            specific to the publication type.

        Args:
            record (Record): Marc12 record describing the publication.

        Returns:
            bool: ``False`` when a non-conformity is found and can not be
                corrected.

        """
        if self.dbg:
            print "check record"

        try:
            self.check.recover_oai(record, self.harvester.host)

            if self.check.is_bad_oai_used(record):
                self.logs[-1].idle(MSG_IN_DB, record.year())
                return False

            self.check.temporary_record(record)
            self.check.authors(record)
            self.check.my_affiliation(record, self.id_project, self.id_team)
            self.check.collaboration(record)

        except Exception as e:
            self.logs[-1].reject(e, record=record)
            return False

        return True

    def get_record_by_fields(self, oai_url, year, **kwargs):
        """Get database record matching fields values defined
        in the keyword arguments.

        Note:
            This method is required to deal with publication entered by hand
            and found later by an harvester.

        Args:
            oai_url (unicode): the oai_url, *e.g*
                ``http://cds.cern.ch/record/123456``. The origin field
                of the existing database record is update to **oai_url**
                when a match is found.

            year (int): the year of the publication. It is used
                by the search algorithm and by the logger.

        Keyword Args:
            kwargs (unicode): a series of key, value pair where the
                key is the name of a publications database field.

        Returns:
            tuple: ``(id, status)`` which contains the ``id`` of the record.
            The ``id`` is equal to ``None`` when there is no matching.
            The ``status`` is equal to one when the existing record was
            modified zero otherwise.

        """
        if self.dbg:
            print "get existing record by fields"

        # alias
        db = self.db
        logs = self.logs

        # add the publication year to search criteria
        if year:
            kwargs["year"] = year

        # look for an existing record
        rec_id = get_id(db.publications, **kwargs)
        if not rec_id:
            return (None, 0)

        # fix origin field
        publication = db.publications[rec_id]
        ok = publication.origin and publication.origin == oai_url
        if not ok:
            if not self.dry_run:
                publication = dict(origin=oai_url)

            logs[-1].modify(MSG_FIX_ORIGIN, year)
            return (rec_id, 1)

        logs[-1].idle(MSG_IN_DB, year)
        return (rec_id, 0)

    def insert_record(self, record):
        """Insert the record in the database.

        Note:
            This method depend on the type of publications.
            It has to be implemented for each inherited class.

        Args:
            record (Record): Marc12 record describing the publication.

        Returns:
            int: one when the record is inserted / updated in the database,
                zero otherwise.

        """
        return 0

    def process_xml(self, xml):
        """Decode the XML string and insert the corresponding records
        in the database.


        Args:
            xml (unicode): XML string encoding the publication records.
                The format follows the MARC12 standard.

        Raises:
           ToolException: when project, team or category identifier
              are not defined.
           StoreException: when something goes wrong interrogating the
              store.
           Marc12Exception: when something goes wrong decoding the XML
              string return by the store.
           CheckException: when the record has non-conformities.
           Exception: when the python code crashes.

        """
        if self.dbg:
            print "start processing", self.__class__.__name__
            print "decode request"

        if self.dbg:
            print "get harvest parameters"

        # decode the XML request
        self.collection_logs.append(MsgCollection(found=1))
        self.decode_xml(xml)

    def process_url(self, host, collections):
        """Retrieve the XML string from the invenio store and
        insert corresponding records in the database.

        Args:
            host (unicode): host name to query for publications, either
                ``cds.cern.ch`` or ``inspirehep.net``.

            collections (unicode): list of collection to be interrogated.

        Raises:
           StoreException: when something goes wrong interrogating the
              store.
           Marc12Exception: when something goes wrong decoding the XML
              string return by the store.
           CheckException: when the record has non-conformities.
           Exception: when the python code crashes.

        """
        if self.dbg:
            print "process URL search"

        # extend harvester for logs
        self.harvester.host = host
        self.harvester.collections = collections

        store = InvenioStore(host)

        # list of collections
        collections = re.sub(' *, *', ',', collections).split(',')

        # alias
        collection_logs = self.collection_logs
        controller = self.controller
        decode_xml = self.decode_xml
        is_record_in_db = self._is_record_in_db
        logs = self.logs
        project = self.db.projects[self.id_project].project

        # extract the list of publications from the store for each collection
        # the search is perform on a range of creation date
        # if not defined all element are return
        #
        # The method uses here minimise the memory usage
        # on the server as well as on the client side

        for collection in collections:

            # log collection information
            # A collection is identified as "Project Controller collection"
            title = "%s / %s / %s" % (project, controller, collection)
            collection_logs.append(MsgCollection(title=title))

            # search record in the harvester repository
            kwargs = self._search_parameters(collection)

            try:
                rec_ids = store.get_ids(**kwargs)

            except Exception as error:
                collection_logs[-1].url = store.last_search_url()
                collection_logs[-1].error = error
                continue

            collection_logs[-1].url = store.last_search_url()
            collection_logs[-1].found = len(rec_ids)

            if not rec_ids:
                continue

            if self.dbg:
                print '%i records found in %s' % (len(rec_ids), collection)

            for rec_id in rec_ids:

                if self.dbg:
                    print "\nprocessing record", rec_id

                try:
                    db_id = is_record_in_db(title, host=host, rec_id=rec_id)
                    if db_id:
                        if self.dbg:
                            print "record in db", rec_id, "->", db_id
                        continue

                    xml = store.get_record(rec_id)
                    decode_xml(xml)

                except Exception as e:
                    print traceback.format_exc()
                    url = OAI_URL % (host, rec_id)
                    logs.append(Msg(harvester=self.harvester,
                                    collection=title,
                                    record_id=rec_id,
                                    title=url))
                    logs[-1].reject(e)

    def decode_xml(self, xml):
        """Decode the MARC XML string and insert records in the database.

        Args:
            xml (unicode): MARC XML string

        """

        if self.dbg:
            print "process xml record"

        # alias
        db = self.db
        check_record = self.check_record
        insert_record = self.insert_record
        is_record_in_db = self._is_record_in_db
        logs = self.logs

        # NOTE
        # BaseException and inherited class
        # are caught by the previous stage
        records = self.marc12(xml)

        # process individual record
        for record in records:

            if self.dbg:
                print "record decoded"
                print record.title()

            # reject the record using the secondary OAI
            # require to cover the case:
            # - scan store A. Only OAI_A is defined.
            # - later scan store B. OAI_B and OAI_A are found.
            rec_id = is_record_in_db(record.title(),
                                     oai_url=record.secondary_oai_url())
            if rec_id:
                # update the oai_url for later uses.
                if not self.dry_run:
                    tp = (record.secondary_oai_url(), record.primary_oai_url())
                    oai_url = "%s, %s" % tp
                    db.publications[rec_id] = dict(origin=oai_url)

                if self.dbg:
                    print "record in db (secondary oai) ->", rec_id

                continue

            # start the log for the record
            logs.append(Msg(harvester=self.harvester,
                            collection=self.collection_logs[-1].title,
                            record_id=record.id(),
                            title=record.title()))

            # check that the record is well formed
            # repair non-conformity as far as possible
            if not check_record(record):
                if self.dbg:
                    print "record rejected", logs[-1].txt
                continue

            if self.dbg:
                print "insert record in the database"

            # insert the record in the database
            insert_record(record)

            if self.dbg:
                print logs[-1].action.upper(), logs[-1].txt

    def report(self):
        """Build the processing report.

        Returns:
            dict:
                * ``collection_logs`` list of :class:MsgCollection
                * ``controller`` unicode
                * ``logs`` list of :class:Msg
                * ``selector`` :py:class:plugin_dbui.Selector

        """

        return dict(collection_logs=self.collection_logs,
                    controller=self.controller,
                    logs=self.logs)

    def search_collaboration(self, value):
        """Get the database collaboration identifier using synonyms.

        Args:
            value (unicode): the name of the collaboration.

        Returns:
            int:
                * the id of the collaboration record.
                * UNDEF_ID if value is not defined.

        Raises:
            ToolException: when more than one synonym is found or when the
                collaboration is not defined.

        """

        return search_synonym(self.db.collaborations, "collaboration", value)

    def search_country(self, value):
        """Get the database country identifier using synonyms.

        Args:
            value (unicode): the name of the country.

        Returns:
            int:
                * the id of the country record.
                * UNDEF_ID if value is not defined.

        Raises:
            ToolException: when more than one synonym is found ot when
                the country is not defined.

        """
        return search_synonym(self.db.countries, "country", value)

    def search_publisher(self, value):
        """Get the database publisher identifier using synonyms.

        Args:
            value (unicode): the abbreviation of the publisher.

        Returns:
            int:
                * the id of the publisher record.
                * UNDEF_ID if value is not defined.

        Raises:
            ToolException: when more than one synonym is found or when
                the publisher is not defined.

        """
        return search_synonym(self.db.publishers, "abbreviation", value)