Commit 4020bbb7 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

break invenio_tools/__init__.py in small pieces.

parent 5b1fc117
This diff is collapsed.
# -*- coding: utf-8 -*-
""" invenio_tools.base
"""
import re
OAI_URL = "http://%s/record/%s"
REG_OAI = re.compile('oai:([a-z\.]+):([\d]+)')
REG_YEAR = re.compile("(\d{4})")
This diff is collapsed.
# -*- coding: utf-8 -*-
""" invenio_tools.exception
"""
class ExceptionUTF8(Exception):
"""Exception in which unicode arguments are encoded as a string.
"""
def __init__(self, *args):
args = (list(args) if isinstance(args, tuple) else args)
for i in range(len(args)):
if isinstance(args[i], unicode):
args[i] = args[i].encode('utf-8')
Exception.__init__(self, *args)
class CdsException(ExceptionUTF8): pass
class CheckException(ExceptionUTF8): pass
class Marc12Exception(ExceptionUTF8): pass
class RecordException(ExceptionUTF8): pass
class XmlException(ExceptionUTF8): pass
# -*- coding: utf-8 -*-
""" invenio_tools.inveniostore
"""
import httplib
import json
import re
import time
import urllib
from gluon import current
from exception import CdsException
CDS_SEARCH_KEYS = ('req', 'cc', 'c', 'ec', 'p', 'f', 'rg', 'sf', 'so', 'sp',
'rm', 'of', 'ot', 'as', 'p1', 'f1', 'm1', 'op1', 'p2', 'f2',
'm2', 'op2', 'p3', 'f3', 'm3', 'sc', 'jrec', 'recid',
'recidb', 'sysno', 'id', 'idb', 'sysnb', 'action', 'd1',
'd1y', 'd1m', 'd1d', 'd2', 'd2y', 'd2m', 'd2d', 'dt',
'verbose', 'ap', 'ln', 'ec')
MSG_HTTP_ERROR = current.T("HTTP Error", lazy=False)
MSG_NO_IDS = current.T("Invalid list of record identifiers", lazy=False)
MSG_WRONG_KEYWORD = current.T("Invalid keyword argument", lazy=False)
# maximum number of ids to be collected at once.
# The value of 200 is the maximum value authorized using cds.cern.ch
N_IDS = 200
REG_IDS_OK = re.compile("^\[[\d, ]*\]$")
class InvenioStore(object):
"""Class to dialogue with U{invenio<http://invenio-software.org/>} store.
In the dialogue:
- the request is an URL
- the response is an XML string which is compliant with the
U{MARC<http://www.loc.gov/marc/>} standard.
"""
def __init__(self, host='cds.cern.ch'):
"""
@type host: str
@param host: domain name for the store
"""
self._host = host
self._url = None
self._try = 0
def interogate(self, url):
"""Interrogate the store using the C{url}.
@type url: unicode
@param url: URL string
@rtype: unicode
@return: the HTTP response
@raise CdsException: when the server return an HTTP error
five consecutive time.
"""
self._url = url
self._try += 1
fi = urllib.urlopen(url)
code = fi.getcode()
data = fi.read()
# the server is busy or return error wait one minute an retry.
# the number of trial is limited to 5
if code == httplib.SERVICE_UNAVAILABLE or code != httplib.OK:
if self._try == 5:
raise CdsException("%s %s" % (MSG_HTTP_ERROR, code))
time.sleep(60)
self.interogate(url)
return data
def get_ids(self, **kwargs):
"""Return a list of record C{id} for records matching search criteria.
Search criteria are defined by the keywords arguments:
@type cc: str
@keyword cc: current collection (e.g. "ATLAS Papers").
The collection the user started to search/browse from.
@type p: str
@keyword p: pattern to search for (e.g. "ellis and muon or kaon").
@type f: str
@keyword f: field to search within (e.g. "author").
@type p1: str
@keyword p1: first pattern to search for in the advanced search
interface. Much like C{p}.
@type f1: str
@keyword f1: first field to search within in the advanced search
interface. Much like C{f}.
@type m1: str
@keyword m1: first matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
@type op1: str
@keyword op1: first operator, to join the first and the second unit
in the advanced search interface. ("a" add, "o" or, "n" not).
@type p2: str
@keyword p2: second pattern to search for in the advanced search
interface. Much like C{p}.
@type f2: str
@keyword f2: second field to search within in the advanced search
interface. Much like C{f}.
@type m2: str
@keyword m2: second matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
@type op2: str
@keyword op2: second operator, to join the second and the third unit
in the advanced search interface. ("a" add, "o" or,"n" not).
@type p3: str
@keyword p3: third pattern to search for in the advanced search
interface. Much like C{p}.
@type f3: str
@keyword f3: third field to search within in the advanced search
interface. Much like C{f}.
@type m3: str
@keyword m3: third matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
The complete list of keyword arguments can be found at
U{http://invenio-demo.cern.ch/help/hacking/search-engine-api}.
@rtype: list
@returns: a list of numbers.
The list is empty when the request failed on the server.
@raise CdsException: when a keyword argument is invalid or
when the server return an invalid list of ids.
"""
for k in kwargs:
if k not in CDS_SEARCH_KEYS:
raise CdsException(MSG_WRONG_KEYWORD, k)
ids = []
scan = True
# NOTE: the number of ids which can be collected at once is limited
# to 10 on cds.cern.ch since the invenio version 1.1.3.1106 (Jun 2014)
# Therefore to recuperate the complete list of ids we have to get them
# by block of 200. The later is the maximum value allowed by cds.
# We use the parameter rg and jrec to steer the scan.
# Thy have no effect on inspirehep.net.
kwargs['of'] = 'id'
kwargs['rg'] = N_IDS
kwargs['jrec'] = -N_IDS
while scan:
self._try = 0
kwargs['jrec'] += N_IDS
params = urllib.urlencode(kwargs)
url = 'http://%s/search?%s' % (self._host, params)
rep = self.interogate(url)
# check that the list of ids is well form
# [1291068, 1352722, 1376692, 1454870, 1492807] or [1493820] or []
if REG_IDS_OK.match(rep):
li = json.loads(rep)
ids.extend(li)
else:
raise CdsException(MSG_NO_IDS)
if len(li) != N_IDS:
scan = False
return ids
def get_record(self, id):
"""Retrieve a record defined by its C{id} in the invenio store.
@type id: int
@param id:
@rtype: unicode
@return: the XML string is compliant with
the U{MARC<http://www.loc.gov/marc/>} standard.
Use L{Marc12.__call__} to decode it.
@raise CdsException: when the server return an HTTP error.
"""
self._try = 0
url = 'http://%s/record/%s/export/xm' % (self._host, id)
return self.interogate(url)
def last_search_url(self):
"""Get the URL used in the last search.
@rtype: unicode
"""
return self._url
def search(self, **kwargs):
"""Return records matching the keyword arguments.
@type req: str
@keyword req: mod_python Request class instance.
@type cc: str
@keyword cc: current collection (e.g. "ATLAS").
The collection the user started to search/browse from.
@type c: str
@keyword c: collection list (e.g. ["Theses", "Books"]).
The collections user may have selected/deselected when
starting to search from C{cc}.
@type ec: str
@keyword ec: external collection list (e.g. ['CiteSeer', 'Google']).
The external collections may have been selected/deselected by the
user.
@type p: str
@keyword p: pattern to search for (e.g. "ellis and muon or kaon").
@type f: str
@keyword f: field to search within (e.g. "author").
@type rg: int
@keyword rg: records in groups of (e.g. "10").
Defines how many hits per collection in the search results page are
displayed.
@type sf: str
@keyword sf: sort field (e.g. "title").
@type so: str
@keyword so: sort order ("a"=ascending, "d"=descending).
@type sp: str
@keyword sp: sort pattern (e.g. "CERN-") -- in case there are more
values in a sort field, this argument tells which one to prefer
@type rm: str
@keyword rm: ranking method (e.g. "jif").
Defines whether results should be ranked by some known ranking method.
@type of: str
@keyword of: output format (e.g. "hb").
Usually starting "h" means HTML output (and "hb" for HTML brief,
"hd" for HTML detailed), "x" means XML output, "t" means plain text
output, "id" means no output at all but to return list of recIDs found.
(Suitable for high-level API.)
@type ot: str
@keyword ot: output only these MARC tags (e.g. "100,700,909C0b").
Useful if only some fields are to be shown in the
output, e.g. for library to control some fields.
@type as: int
@keyword as: advanced search ("0" means no, "1" means yes).
Whether search was called from within the advanced search
interface.
@type p1: str
@keyword p1: first pattern to search for in the advanced search
interface. Much like C{p}.
@type f1: str
@keyword f1: first field to search within in the advanced search
interface. Much like C{f}.
@type m1: str
@keyword m1: first matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact phrase,
"p" partial phrase, "r" regular expression).
@type op1: str
@keyword op1: first operator, to join the first and the second unit
in the advanced search interface. ("a" add, "o" or, "n" not).
@type p2: str
@keyword p2: second pattern to search for in the advanced search
interface. Much like C{p}.
@type f2: str
@keyword f2: second field to search within in the advanced search
interface. Much like C{f}.
@type m2: str
@keyword m2: second matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact phrase,
"p" partial phrase, "r" regular expression).
@type op2: str
@keyword op2: second operator, to join the second and the third unit
in the advanced search interface. ("a" add, "o" or, "n" not).
@type p3: str
@keyword p3: third pattern to search for in the advanced search
interface. Much like C{p}.
@type f3: str
@keyword f3: third field to search within in the advanced search
interface. Much like C{f}.
@type m3: str
@keyword m3: third matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact phrase,
"p" partial phrase, "r" regular expression).
@type sc: int
@keyword sc: split by collection ("0" no, "1" yes).
Governs whether we want to present the results in a single huge list,
or splitted by collection.
@type jrec: int
@keyword jrec: jump to record (e.g. "234").
Used for navigation inside the search results.
@type recid: int
@keyword recid: display record ID (e.g. "20000").
Do not search/browse but go straight away to the Detailed
record page for the given recID.
@type recidb: int
@keyword recidb: display record ID bis (e.g. "20010").
If greater than 'recid', then display records from recid to recidb.
Useful for example for dumping records from the
database for reformatting.
@type sysno: str
@keyword sysno: display old system SYS number (e.g. "").
If you migrate to Invenio from another system, and store your
old SYS call numbers, you can use them instead of recid if you wish so.
@type id: int
@keyword id: the same as recid, in case recid is not set.
For backwards compatibility.
@type idb: int
@keyword idb: the same as recid, in case recidb is not set.
For backwards compatibility.
@type sysnb: str
@keyword sysnb: the same as sysno, in case sysno is not set.
For backwards compatibility.
@type action: str
@keyword action: action to do. "SEARCH" for searching,
"Browse" for browsing. Default is to search.
@type d1: str
@keyword d1: first datetime in full YYYY-mm-dd HH:MM:DD format
(e.g. "1998-08-23 12:34:56"). Useful for search limits
on creation/modification date (see 'dt' argument below).
Note that 'd1' takes precedence over d1y, d1m, d1d if these
are defined.
@type d1y: int
@keyword d1y: first date's year (e.g. "1998").
Useful for search limits on creation/modification date.
@type d1m: int
@keyword d1m: first date's month (e.g. "08").
Useful for search limits on creation/modification date.
@type d1d: int
@keyword d1d: first date's day (e.g. "23").
Useful for search limits on creation/modification date.
@type d2: str
@keyword d2: second datetime in full YYYY-mm-dd HH:MM:DD format
(e.g. "1998-09-02 12:34:56"). Useful for search limits
on creation/modification date (see 'dt' argument below).
Note that 'd2' takes precedence over d2y, d2m, d2d
if these are defined.
@type d2y: int
@keyword d2y: second date's year (e.g. "1998").
Useful for search limits on creation/modification date.
@type d2m: int
@keyword d2m: second date's month (e.g. "09").
Useful for search limits on creation/modification date.
@type d2d: int
@keyword d2d: second date's day (e.g. "02").
Useful for search limits on creation/modification date.
@type dt: str
@keyword dt: first and second date's type (e.g. "c").
Specifies whether to search in creation dates ("c") or in
modification dates ("m"). When dt is not set and d1*
and d2* are set, the default is "c".
@type verbose: int
@keyword verbose: verbose level (0=min, 9=max).
Useful to print some internal information on the searching process
in case something goes wrong.
@type ap: int
@keyword ap: alternative patterns (0=no, 1=yes).
In case no exact match is found, the search engine can try
alternative patterns e.g. to replace non-alphanumeric characters by
a boolean query. ap defines if this is wanted.
@type ln: str
@keyword ln: language of the search interface (e.g. "en").
Useful for internationalization.
The keyword arguments are those of the invenio web interface.
Details are in
U{http://invenio-demo.cern.ch/help/hacking/search-engine-api}.
@rtype: unicode
@return: The format of the string (HTML, XML) depend on the keyword C{of}.
For MARC12 format use C{xm}.
@deprecated: the method L{get_ids} coupled with L{get_record} are much more
efficient.
"""
for k in kwargs:
if k not in CDS_SEARCH_KEYS:
raise CdsException(MSG_WRONG_KEYWORD, k)
kwargs['action_search'] = 'Search'
params = urllib.urlencode(kwargs)
url = 'http://%s/search?%s' % (self._host, params)
return self.interogate(url)
def search_year(self, collection, year, of='xm', rg=10, so='d'):
"""Search records for given C{collection} and for a given C{year}.
@type collection: str
@param collection:
@type year: int
@param year:
@type of: str
@param of: output format (e.g. "hb").
Usually starting "h" means HTML output (and "hb" for HTML brief,
"hd" for HTML detailed), "x" means XML output,
"xm" means MARC XML format, "t" means plain text output,
"id" means no output at all but to return list of recIDs found.
(Suitable for high-level API.)
@type rg: int
@param rg: limit on the number of output records.
@type so: str
@param so: sort order ("a"=ascending, "d"=descending).
@rtype: unicode
@return: The format of the string HTML, XML depends on the
keyword C{of}. Use C{xm} for MARC XML.
"""
return self.search(c=collection,
f='year',
ln='en',
of=of,
p=year,
rg=rg,
sf='year',
so=so)
# -*- coding: utf-8 -*-
""" invenio_tools.marc12
"""
import re
from exception import Marc12Exception
from gluon import current
from record import Record
from xml.dom.minidom import parseString
MSG_WELL_FORMED_XML = current.T("Reject XML is not well formed", lazy=False)
REG_INT = re.compile("^\d+$")
class Marc12(object):
"""Decode the XML string encoded with the
U{MARC<http://www.loc.gov/marc>} format.
The main method L{__call__} analyses the XML string
which has the follwing structure::
<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record>
<controlfield tag="001">1540265</controlfield>
<controlfield tag="005">20130410235250.0</controlfield>
<datafield tag="024" ind1="8" ind2=" ">
<subfield code="a">oai:cds.cern.ch:1540265</subfield>
<subfield code="p">cerncds:FULLTEXT</subfield>
<subfield code="p">cerncds:CERN:FULLTEXT</subfield>
<subfield code="p">cerncds:CERN</subfield>
</datafield>
...
</record>
<record>
...
</record>
</collection>
It returns a L{Record} object which behave like a dictionary::
record[field][subfield] = value(s)
where the C{field} correspond to the I{datafield tag} and the
C{subfield} to the I{subfield code}.
"""
def _clean_record(self, record):
"""Internal tool to clean the record.
concatenate the following dictionary::
record[field] = [dict(subfield1=val1), dict(subfield2=val2), dict(subfield3=val3),...]
record[field] = [dict(subfield1=val1), dict(subfield2=val2, subfield3=val3),...]
into a single one::
record[field] = dict1(subfield1=val1, subfield2=val2, subfield3=val3)
@type record: Record
@param record:
"""
for field in record:
if not isinstance(record[field], list):
continue
nkeys = [len(di) for di in record[field]]
# several dictionary with more than one nkeys
# don't know how to treat that case
if max(nkeys) > 1 and nkeys.count(max(nkeys)) > 1:
continue
# merge single entity dict in one big dict
# works when all the nkeys are different
# otherwise don't know what to do
if max(nkeys) == 1:
keys = []
for di in record[field]:
keys.extend(di.iterkeys())
# in a set duplicate entries are removed
# the next statement is true when all keys are different
if len(keys) == len(set(keys)):
di = record[field][0]
for i in range(1, len(record[field])):
for (k, v) in record[field][i].iteritems():
di[k] = v
record[field] = di