inveniostore.py

""" store_tools.inveniostore

"""
import requests

from .exception import CdsException
from requests.adapters import HTTPAdapter
from builtins import getattr

CDS_SEARCH_KEYS = ("req", "cc", "c", "ec", "p", "f", "rg", "sf", "so", "sp",
                   "rm", "of", "ot", "as", "p1", "f1", "m1", "op1", "p2", "f2",
                   "m2", "op2", "p3", "f3", "m3", "sc", "jrec", "recid",
                   "recidb", "sysno", "id", "idb", "sysnb", "action", "d1",
                   "d1y", "d1m", "d1d", "d2", "d2y", "d2m", "d2d", "dt",
                   "verbose", "ap", "ln", "ec")

MSG_HTTP_DECODE = "Fail to decode HTTP response"
MSG_HTTP_ERROR = "HTTP Error"
MSG_INVALID_RESPONSE = "Invalid response"
MSG_NO_IDS = "Invalid list of record identifiers"
MSG_NOT_IMPLEMENTED = "Method '%s' not implemented for store %s and shelf %s"
MSG_WRONG_KEYWORD = "Invalid keyword argument"

# maximum number of identifiers to be collected at once.
# The value of 200 is the maximum value authorised using cds.cern.ch
N_IDS = 200


class InvenioStore(object):
    """Class to dialogue with `invenio <http://invenio-software.org/>`_ store.

    The class provides methods to request:
        * a list of identifier satisfying search criteria.
        * a record identified by its id.

    Args:
        host (str):
            possible values are ``cds``, ``cds.cern.ch``,``inspirehep``
            or ``inspirehep.net``

        api_search (str):

        api_record (str):

        max_retries (int):

        shelf (str):
            section of the store containing records. It depends on the host.
            Possible values are ``None``, ``literature``, ``conferences``
            and ``institutions``

             +----------------+--------------+-----------------------------+
             | host           | shelf        | base API                    |
             +----------------+--------------+-----------------------------+
             | cds.cern.ch    | None         | https://cds.cern.ch/        |
             +----------------+--------------+-----------------------------+
             | inspirehep.net | None         | https://old.inspirehep.net/ |
             | inspirehep.net | literature   | https://old.inspirehep.net/ |
             | inspirehep.net | conferences  | https://inspirehep.net/     |
             | inspirehep.net | institutions | https://old.inspirehep.net/ |
             +----------------+--------------+-----------------------------+

    """

    def __init__(self,
                 api_record=None,
                 api_search=None,
                 host=None,
                 max_retries=3,
                 shelf=None):

        self._api_search = api_search
        self._api_record = api_record
        self._host = host
        self._shelf = shelf
        self._url = None

        # start a session, a persistent connection with the server
        # let the session handle the number of retry
        session = requests.Session()
        session.mount(f"https://{host}", HTTPAdapter(max_retries=max_retries))
        self._session = session

    def __del__(self):
        # close the session
        if getattr(self, "_session", None) is not None:
            self._session.close()

    def get_field(self, rec_id, fieldname):
        """Retrieve the field value for the record identified by
        its *record id*.

        Note:
            The method is implemented for store, shelf pairs
            relying on ``cds.cern.ch`` and ``old.inspirehep.net``.

        Args:
            rec_id (int):
                record identifier in the store.

            fieldname (str):
                name of the field in the JSON record.

        Returns:
            * int
            * str
            * None when the field is not found

        Raises:
            CdsException:
                * method is not implemented for all store, shelf pairs.
                  It works for those relying on cds.cern.ch and
                  old.inspirehep.net
                * the server return an HTTP error.
                * JSON object could not be decoded.

        """
        host = self._host
        if host not in  ("cds.cern.ch", "old.inspirehep.net"):
            msg = MSG_NOT_IMPLEMENTED % ("get_field", host, self._shelf)
            raise CdsException(msg)

        url = f"{self._api_record}/{rec_id}"

        rep = self.interrogate(url, timeout=60, of="recjson", ot=fieldname)

        try:
            obj = rep.json()

        except ValueError:
            raise CdsException(MSG_HTTP_DECODE)

        if isinstance(obj, list) and len(obj) == 1:
            return obj[0][fieldname]

        return None

    def get_ids(self, **kwargs):
        """Return a list of *record id* matching search criteria.

        Note:
            The method is implemented for store, shelf pairs
            relying on ``cds.cern.ch`` and ``old.inspirehep.net``.

        Keyword Args:

            The keyword arguments are those of the invenio search engine and
            they depend on the version of invenio.

            See https://gitlab.in2p3.fr/limbra/limbra/-/blob/master/modules/store_tools/README.md
            for more information.

        Returns:
            list:
                * A list of numbers.
                * The list is empty when the request failed on the server.

        Raises:
            CdsException::

                * Method not implemented for all store, shelf pairs.
                  It works for those relying on cds.cern.ch and
                  old.inspirehep.net
                * keyword argument is invalid;
                * the server return an HTTP error;
                * JSON object can't be decoded;
                * not well formed list of ids.

        """
        host = self._host
        if host not in  ("cds.cern.ch", "old.inspirehep.net"):
            msg = MSG_NOT_IMPLEMENTED % ("get_ids", host, self._shelf)
            raise CdsException(msg)

        for k in kwargs:
            if k not in CDS_SEARCH_KEYS:
                raise CdsException(MSG_WRONG_KEYWORD, k)

        ids = []
        scan = True

        # NOTE: the number of ids which can be collected at once is limited
        # to 10 on cds.cern.ch since the invenio version 1.1.3.1106 (Jun 2014)
        # Therefore to get the complete list of ids we have to scan them
        # by block of 200. The later is the maximum value allowed by cds.
        # We use the parameter rg and jrec to steer the scan.
        # They have no effect on inspirehep.net.

        kwargs["of"] = "id"
        kwargs["rg"] = N_IDS
        kwargs["jrec"] = -N_IDS

        while scan:
            kwargs["jrec"] += N_IDS

            rep = self.interrogate(self._api_search, timeout=30, **kwargs)

            try:
                li = rep.json()

                # check that the list of identifier is well form
                # [1291068, 1352722, 1376692] or [1493820] or []
                if len(list(filter(lambda x: not isinstance(x, int), li))) > 0:
                    raise CdsException(MSG_NO_IDS)

                ids.extend(li)

            # trigger when the JSON object cannot be decoded
            except ValueError as e:
                raise CdsException(e)

            if len(li) != N_IDS:
                scan = False

        return ids

    def get_record(self, rec_id):
        """Retrieve a record defined by its *record id*.

        Args:
            rec_id (int):
                record identifier in the store.

        Returns:
            dict:
                the record data (recjson).

        Raises:
            CdsException:

                * the server return an HTTP error.
                * JSON object could not be decoded.
                * more than one record

        """
        url = f"{self._api_record}/{rec_id}"

        kwargs = {}
        if self._host in ("cds.cern.ch", "old.inspirehep.net"):
            kwargs = {"of": "recjson"}

        rep = self.interrogate(url, timeout=30, **kwargs)

        try:
            obj = rep.json()

        except ValueError:
            raise CdsException(MSG_HTTP_DECODE)

        if isinstance(obj, dict):
            return obj

        if isinstance(obj, list) and len(obj) == 1:
            return obj[0]

        raise CdsException(MSG_INVALID_RESPONSE)

    def interrogate(self, url, timeout=10, **kwargs):
        """Interrogate the store using the *URL*.
        It is retry several time when the service is not available.

        Args:
            url (str):
                the URL string depends on the store and on the invenio
                version which is running, *e.g.*::

                    * ``https://cds.cern.ch/record/123456/of=recjson``
                    * ``https://cds.cern.ch/search?of=id&....

            timeout (float):
                timeout for the HTTP request

        Keyword Args:

            The keyword arguments are those of the invenio search engine and
            they depend on the version of invenio.

            See https://gitlab.in2p3.fr/limbra/limbra/-/blob/master/modules/store_tools/README.md
            for more information.

        Returns:
            requests.Response:

        Raises:
            RequestException:
                something went wrong within the HTTP dialog

        """
        self._url = url

        r = self._session.get(url, timeout=timeout, params=kwargs)
        r.raise_for_status()

        return r

    def last_search_url(self):
        """
        Returns:
            str:
                the URL used in the last search.

        """
        return self._url

    def search(self, query, **kwargs):
        """Return a list of *JSON record* matching search criteria.

        Note:
            The method is implemented for store, shelf pairs
            relying on ``inspirehep.net``.

        Args:
            query (str):
                query for the inspirehep store.
                Use the syntax of the web interface or elasticsearch one.

        Keyword Args:

            elasticsearch keywords

        Returns:
            * list of JSON records

        Raises:
            CdsException:

                * the server return an HTTP error.
                * JSON object could not be decoded.

        """
        url = f"{self._api_search}{query}"

        rep = self.interrogate(url, timeout=30, **kwargs)

        try:
            obj = rep.json()

        except ValueError:
            raise CdsException(MSG_HTTP_DECODE)

        # the response is a dict with 3 keys: 'hits', 'links', 'sort_options'
        # the hits section is a dict with 2 keys: hits (list), total (int)
        try:
            return obj["hits"]["hits"]

        except (KeyError, TypeError):
            raise CdsException(MSG_INVALID_RESPONSE)