inveniostore.py 16.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
# -*- coding: utf-8 -*-
""" invenio_tools.inveniostore

"""
import httplib
import json
import re
import time
import urllib

from gluon import current
from exception import CdsException


CDS_SEARCH_KEYS = ('req', 'cc', 'c', 'ec', 'p', 'f', 'rg', 'sf', 'so', 'sp',
                   'rm', 'of', 'ot', 'as', 'p1', 'f1', 'm1', 'op1', 'p2', 'f2',
                   'm2', 'op2', 'p3', 'f3', 'm3', 'sc', 'jrec', 'recid',
                   'recidb', 'sysno', 'id', 'idb', 'sysnb', 'action', 'd1',
                   'd1y', 'd1m', 'd1d', 'd2', 'd2y', 'd2m', 'd2d', 'dt',
                   'verbose', 'ap', 'ln', 'ec')

MSG_HTTP_ERROR = current.T("HTTP Error", lazy=False)
MSG_NO_IDS = current.T("Invalid list of record identifiers", lazy=False)
MSG_WRONG_KEYWORD = current.T("Invalid keyword argument", lazy=False)

# maximum number of ids to be collected at once.
# The value of 200 is the maximum value authorized using cds.cern.ch
N_IDS = 200

REG_IDS_OK = re.compile("^\[[\d, ]*\]$")


class InvenioStore(object):
    """Class to dialogue with U{invenio<http://invenio-software.org/>} store.

    In the dialogue:

        - the request is an URL
        - the response is an XML string which is compliant with the
          U{MARC<http://www.loc.gov/marc/>} standard.

    """
    def __init__(self, host='cds.cern.ch'):
        """
        @type host: str
        @param host: domain name for the store

        """
        self._host = host
        self._url = None
        self._try = 0

    def interogate(self, url):
        """Interrogate the store using the C{url}.

        @type url: unicode
        @param url: URL string

        @rtype: unicode
        @return: the HTTP response

        @raise CdsException: when the server return an HTTP error
        five consecutive time.

        """
        self._url = url
        self._try += 1

        fi = urllib.urlopen(url)

        code = fi.getcode()
        data = fi.read()

        # the server is busy or return error wait one minute an retry.
        # the number of trial is limited to 5
        if code == httplib.SERVICE_UNAVAILABLE or code != httplib.OK:
            if self._try == 5:
                raise CdsException("%s %s" % (MSG_HTTP_ERROR, code))

            time.sleep(60)
            self.interogate(url)

        return data

    def get_ids(self, **kwargs):
        """Return a list of record C{id} for records matching search criteria.

        Search criteria are defined by the keywords arguments:

        @type cc: str
        @keyword cc: current collection (e.g. "ATLAS Papers").
        The collection the user started to search/browse from.

        @type p: str
        @keyword p: pattern to search for (e.g. "ellis and muon or kaon").

        @type f: str
        @keyword f: field to search within (e.g. "author").

        @type p1: str
        @keyword p1: first pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f1: str
        @keyword f1: first field to search within in the advanced search
        interface.  Much like C{f}.

        @type m1: str
        @keyword m1: first matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact
        phrase, "p" partial phrase, "r" regular expression).

        @type op1: str
        @keyword op1: first operator, to join the first and the second unit
        in the advanced search interface.  ("a" add, "o" or, "n" not).

        @type p2: str
        @keyword p2: second pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f2: str
        @keyword f2: second field to search within in the advanced search
        interface.  Much like C{f}.

        @type m2: str
        @keyword m2: second matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact
        phrase, "p" partial phrase, "r" regular expression).

        @type op2: str
        @keyword op2: second operator, to join the second and the third unit
        in the advanced search interface.  ("a" add, "o" or,"n" not).

        @type p3: str
        @keyword p3: third pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f3: str
        @keyword f3: third field to search within in the advanced search
        interface.  Much like C{f}.

        @type m3: str
        @keyword m3: third matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact
        phrase, "p" partial phrase, "r" regular expression).

        The complete list of  keyword arguments can be found at
        U{http://invenio-demo.cern.ch/help/hacking/search-engine-api}.

        @rtype: list
        @returns: a list of numbers.
        The list is empty when the request failed on the server.

        @raise CdsException: when a keyword argument is invalid or
        when the server return an invalid list of ids.

        """
        for k in kwargs:
            if k not in CDS_SEARCH_KEYS:
                raise CdsException(MSG_WRONG_KEYWORD, k)

        ids = []
        scan = True

        # NOTE: the number of ids which can be collected at once is limited
        # to 10 on cds.cern.ch since the invenio version 1.1.3.1106 (Jun 2014)
        # Therefore to recuperate the complete list of ids we have to get them
        # by block of 200. The later is the maximum value allowed by cds.
        # We use the parameter rg and jrec to steer the scan.
        # Thy have no effect on inspirehep.net.

        kwargs['of'] = 'id'
        kwargs['rg'] = N_IDS
        kwargs['jrec'] = -N_IDS

        while scan:
            self._try = 0
            kwargs['jrec'] += N_IDS

            params = urllib.urlencode(kwargs)
            url = 'http://%s/search?%s' % (self._host, params)

            rep = self.interogate(url)

            # check that the list of ids is well form
            # [1291068, 1352722, 1376692, 1454870, 1492807] or [1493820] or []
            if REG_IDS_OK.match(rep):
                li = json.loads(rep)
                ids.extend(li)

            else:
                raise CdsException(MSG_NO_IDS)

            if len(li) != N_IDS:
                scan = False

        return ids

    def get_record(self, id):
        """Retrieve a record defined by its C{id} in the invenio store.

        @type id: int
        @param id:

        @rtype: unicode
        @return: the XML string is compliant with
        the U{MARC<http://www.loc.gov/marc/>} standard.
        Use L{Marc12.__call__} to decode it.

        @raise CdsException: when the server return an HTTP error.

        """
        self._try = 0
        url = 'http://%s/record/%s/export/xm' % (self._host, id)
        return self.interogate(url)

    def last_search_url(self):
        """Get the URL used in the last search.

        @rtype: unicode

        """
        return self._url

    def search(self, **kwargs):
        """Return records matching the keyword arguments.

        @type req: str
        @keyword req: mod_python Request class instance.

        @type cc: str
        @keyword cc: current collection (e.g. "ATLAS").
        The collection the user started to search/browse from.

        @type c: str
        @keyword c: collection list (e.g. ["Theses", "Books"]).
        The collections user may have selected/deselected when
        starting to search from C{cc}.

        @type ec: str
        @keyword ec: external collection list (e.g. ['CiteSeer', 'Google']).
        The external collections may have been selected/deselected by the
        user.

        @type p: str
        @keyword p: pattern to search for (e.g. "ellis and muon or kaon").

        @type f: str
        @keyword f: field to search within (e.g. "author").

        @type rg: int
        @keyword rg: records in groups of (e.g. "10").
        Defines how many hits per collection in the search results page are
        displayed.

        @type sf: str
        @keyword sf: sort field (e.g. "title").

        @type so: str
        @keyword so: sort order ("a"=ascending, "d"=descending).

        @type sp: str
        @keyword sp: sort pattern (e.g. "CERN-") -- in case there are more
        values in a sort field, this argument tells which one to prefer

        @type rm: str
        @keyword rm: ranking method (e.g. "jif").
        Defines whether results should be ranked by some known ranking method.

        @type of: str
        @keyword of: output format (e.g. "hb").
        Usually starting "h" means HTML output (and "hb" for HTML brief,
        "hd" for HTML detailed), "x" means XML output, "t" means plain text
        output, "id" means no output at all but to return list of recIDs found.
        (Suitable for high-level API.)

        @type ot: str
        @keyword ot: output only these MARC tags (e.g. "100,700,909C0b").
        Useful if only some fields are to be shown in the
        output, e.g. for library to control some fields.

        @type as: int
        @keyword as: advanced search ("0" means no, "1" means yes).
        Whether search was called from within the advanced search
        interface.

        @type p1: str
        @keyword p1: first pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f1: str
        @keyword f1: first field to search within in the advanced search
        interface.  Much like C{f}.

        @type m1: str
        @keyword m1: first matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact phrase,
        "p" partial phrase, "r" regular expression).

        @type op1: str
        @keyword op1: first operator, to join the first and the second unit
        in the advanced search interface.  ("a" add, "o" or, "n" not).

        @type p2: str
        @keyword p2: second pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f2: str
        @keyword f2: second field to search within in the advanced search
        interface.  Much like C{f}.

        @type m2: str
        @keyword m2: second matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact phrase,
        "p" partial phrase, "r" regular expression).

        @type op2: str
        @keyword op2: second operator, to join the second and the third unit
        in the advanced search interface.  ("a" add, "o" or, "n" not).

        @type p3: str
        @keyword p3: third pattern to search for in the advanced search
        interface.  Much like C{p}.

        @type f3: str
        @keyword f3: third field to search within in the advanced search
        interface.  Much like C{f}.

        @type m3: str
        @keyword m3: third matching type in the advanced search interface.
        ("a" all of the words, "o" any of the words, "e" exact phrase,
        "p" partial phrase, "r" regular expression).

        @type sc: int
        @keyword sc: split by collection ("0" no, "1" yes).
        Governs whether we want to present the results in a single huge list,
        or splitted by collection.

        @type jrec: int
        @keyword jrec: jump to record (e.g. "234").
        Used for navigation inside the search results.

        @type recid: int
        @keyword recid: display record ID (e.g. "20000").
        Do not search/browse but go straight away to the Detailed
        record page for the given recID.

        @type recidb: int
        @keyword recidb: display record ID bis (e.g. "20010").
        If greater than 'recid', then display records from recid to recidb.
        Useful for example for dumping records from the
        database for reformatting.

        @type sysno: str
        @keyword sysno: display old system SYS number (e.g. "").
        If you migrate to Invenio from another system, and store your
        old SYS call numbers, you can use them instead of recid if you wish so.

        @type id: int
        @keyword id: the same as recid, in case recid is not set.
        For backwards compatibility.

        @type idb: int
        @keyword idb: the same as recid, in case recidb is not set.
        For backwards compatibility.

        @type sysnb: str
        @keyword sysnb: the same as sysno, in case sysno is not set.
        For backwards compatibility.

        @type action: str
        @keyword action: action to do.  "SEARCH" for searching,
        "Browse" for browsing.  Default is to search.

        @type d1: str
        @keyword d1: first datetime in full YYYY-mm-dd HH:MM:DD format
        (e.g. "1998-08-23 12:34:56"). Useful for search limits
        on creation/modification date (see 'dt' argument below).
        Note that 'd1' takes precedence over d1y, d1m, d1d if these
        are defined.

        @type d1y: int
        @keyword d1y: first date's year (e.g. "1998").
        Useful for search limits on creation/modification date.

        @type d1m: int
        @keyword d1m: first date's month (e.g. "08").
        Useful for search limits on creation/modification date.

        @type d1d: int
        @keyword d1d: first date's day (e.g. "23").
        Useful for search limits on creation/modification date.

        @type d2: str
        @keyword d2: second datetime in full YYYY-mm-dd HH:MM:DD format
        (e.g. "1998-09-02 12:34:56"). Useful for search limits
        on creation/modification date (see 'dt' argument below).
        Note that 'd2' takes precedence over d2y, d2m, d2d
        if these are defined.

        @type d2y: int
        @keyword d2y: second date's year (e.g. "1998").
        Useful for search limits on creation/modification date.

        @type d2m: int
        @keyword d2m: second date's month (e.g. "09").
        Useful for search limits on creation/modification date.

        @type d2d: int
        @keyword d2d: second date's day (e.g. "02").
        Useful for search limits on creation/modification date.

        @type dt: str
        @keyword dt: first and second date's type (e.g. "c").
        Specifies whether to search in creation dates ("c") or in
        modification dates ("m").  When dt is not set and d1*
        and d2* are set, the default is "c".

        @type verbose: int
        @keyword verbose: verbose level (0=min, 9=max).
        Useful to print some internal information on the searching process
        in case something goes wrong.

        @type ap: int
        @keyword ap: alternative patterns (0=no, 1=yes).
        In case no exact match is found, the search engine can try
        alternative patterns e.g. to replace non-alphanumeric characters by
        a boolean query.  ap defines if this is wanted.

        @type ln: str
        @keyword ln: language of the search interface (e.g. "en").
        Useful for internationalization.

        The keyword arguments are those of the invenio web interface.
        Details are in
        U{http://invenio-demo.cern.ch/help/hacking/search-engine-api}.

        @rtype: unicode
        @return: The format of the string (HTML, XML) depend on the keyword C{of}.
        For MARC12 format use C{xm}.

        @deprecated: the method L{get_ids} coupled with L{get_record} are much more
        efficient.

        """
        for k in kwargs:
            if k not in CDS_SEARCH_KEYS:
                raise CdsException(MSG_WRONG_KEYWORD, k)

        kwargs['action_search'] = 'Search'

        params = urllib.urlencode(kwargs)
        url = 'http://%s/search?%s' % (self._host, params)

        return self.interogate(url)

    def search_year(self, collection, year, of='xm', rg=10, so='d'):
        """Search records for given C{collection} and for a given C{year}.

        @type collection: str
        @param collection:

        @type year: int
        @param year:

        @type of: str
        @param of: output format (e.g. "hb").
        Usually starting "h" means HTML output (and "hb" for HTML brief,
        "hd" for HTML detailed), "x" means XML output,
        "xm" means MARC XML format, "t" means plain text output,
        "id" means no output at all but to return list of recIDs found.
        (Suitable for high-level API.)

        @type rg: int
        @param rg: limit on the number of output records.

        @type so: str
        @param so: sort order ("a"=ascending, "d"=descending).

        @rtype: unicode
        @return: The format of the string HTML, XML depends on the
        keyword C{of}. Use C{xm} for MARC XML.

        """
        return self.search(c=collection,
                           f='year',
                           ln='en',
                           of=of,
                           p=year,
                           rg=rg,
                           sf='year',
                           so=so)