Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
limbra
limbra
Commits
68597c28
Commit
68597c28
authored
Sep 29, 2015
by
LE GAC Renaud
Browse files
Apply PEP-8 rules.
parent
4d708b04
Changes
18
Show whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
230 additions
and
187 deletions
+230
-187
modules/callbacks.py
modules/callbacks.py
+25
-23
modules/check_tools.py
modules/check_tools.py
+49
-35
modules/countries.py
modules/countries.py
+4
-4
modules/filters.py
modules/filters.py
+22
-21
modules/harvest_tools/automaton.py
modules/harvest_tools/automaton.py
+2
-2
modules/harvest_tools/base.py
modules/harvest_tools/base.py
+3
-3
modules/harvest_tools/msgcollection.py
modules/harvest_tools/msgcollection.py
+0
-2
modules/invenio_tools/base.py
modules/invenio_tools/base.py
+7
-6
modules/invenio_tools/checkandfix.py
modules/invenio_tools/checkandfix.py
+18
-18
modules/invenio_tools/exception.py
modules/invenio_tools/exception.py
+20
-5
modules/invenio_tools/inveniostore.py
modules/invenio_tools/inveniostore.py
+4
-4
modules/invenio_tools/iterrecord.py
modules/invenio_tools/iterrecord.py
+14
-5
modules/invenio_tools/record.py
modules/invenio_tools/record.py
+3
-3
modules/invenio_tools/recordconf.py
modules/invenio_tools/recordconf.py
+4
-3
modules/invenio_tools/recordpubli.py
modules/invenio_tools/recordpubli.py
+3
-3
modules/list_postprocessing.py
modules/list_postprocessing.py
+40
-41
modules/regex.py
modules/regex.py
+2
-1
modules/reporting_tools.py
modules/reporting_tools.py
+10
-8
No files found.
modules/callbacks.py
View file @
68597c28
...
...
@@ -13,6 +13,10 @@ from plugin_dbui import (CALLBACK_ERRORS,
get_where_query
)
MSG_DUPLICATE
=
\
"Can't delete this record since several publications refer to it."
def
INHIBIT_CASCADE_DELETE
(
set_records
):
"""Inhibit the delete when publications use the reference field.
...
...
@@ -51,8 +55,7 @@ def INHIBIT_CASCADE_DELETE(set_records):
query
=
(
query
)
&
(
set_records
.
query
)
if
db
(
query
).
count
():
field
.
_table
[
CALLBACK_ERRORS
]
=
\
T
(
"Can't delete this record since several publications refer to it."
)
field
.
_table
[
CALLBACK_ERRORS
]
=
T
(
MSG_DUPLICATE
)
return
True
return
False
...
...
@@ -225,4 +228,3 @@ def INHIBIT_PUBLICATION_UPDATE_ON_OK(s, f):
return
True
return
False
modules/check_tools.py
View file @
68597c28
...
...
@@ -78,7 +78,8 @@ def check_publication(row):
# publication URL
if
row
.
publications
.
publication_url
:
if
'pdf'
not
in
row
.
publications
.
publication_url
:
text
=
T
(
"Check that the publication URL corresponds to a pdf file."
)
text
=
\
T
(
"Check that the publication URL corresponds to a pdf file."
)
li
.
append
(
text
)
# latex syntax
...
...
@@ -224,30 +225,34 @@ def duplicate_article(publication):
ids
=
[]
db
=
current
.
globalenv
[
'db'
]
qcat
=
(
db
.
categories
.
code
==
'ACL'
)
|
(
db
.
categories
.
code
==
'ACLN'
)
categories
=
db
.
categories
publications
=
db
.
pulications
qmain
=
get_where_query
(
db
.
publications
)
qcat
=
(
categories
.
code
==
'ACL'
)
|
(
categories
.
code
==
'ACLN'
)
qpub
=
publications
.
id_publishers
==
publication
[
'id_publishers'
]
qmain
=
get_where_query
(
publications
)
qmain
=
((
qmain
)
&
(
qcat
))
qmain
=
((
qmain
)
&
(
db
.
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
db
.
publications
.
id_publishers
==
publication
[
'id_publishers'
]
))
qmain
=
((
qmain
)
&
(
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
qpub
))
if
'id'
in
publication
and
publication
[
'id'
]:
qmain
=
((
qmain
)
&
(
db
.
publications
.
id
!=
publication
[
'id'
]))
qmain
=
((
qmain
)
&
(
publications
.
id
!=
publication
[
'id'
]))
# title, publishers, volume and pages
query
=
((
qmain
)
&
(
db
.
publications
.
title
==
publication
[
'title'
]))
query
=
((
query
)
&
(
db
.
publications
.
volume
==
publication
[
'volume'
]))
query
=
((
query
)
&
(
db
.
publications
.
pages
==
publication
[
'pages'
]))
query
=
((
qmain
)
&
(
publications
.
title
==
publication
[
'title'
]))
query
=
((
query
)
&
(
publications
.
volume
==
publication
[
'volume'
]))
query
=
((
query
)
&
(
publications
.
pages
==
publication
[
'pages'
]))
extend_ids
(
db
,
query
,
ids
)
# publisher, volume, pages and year
query
=
((
qmain
)
&
(
db
.
publications
.
volume
==
publication
[
'volume'
]))
query
=
((
query
)
&
(
db
.
publications
.
pages
==
publication
[
'pages'
]))
query
=
((
query
)
&
(
db
.
publications
.
year
==
publication
[
'year'
]))
query
=
((
qmain
)
&
(
publications
.
volume
==
publication
[
'volume'
]))
query
=
((
query
)
&
(
publications
.
pages
==
publication
[
'pages'
]))
query
=
((
query
)
&
(
publications
.
year
==
publication
[
'year'
]))
extend_ids
(
db
,
query
,
ids
)
# publisher and title
query
=
((
qmain
)
&
(
db
.
publications
.
title
==
publication
[
'title'
]))
query
=
((
qmain
)
&
(
publications
.
title
==
publication
[
'title'
]))
extend_ids
(
db
,
query
,
ids
)
return
ids
...
...
@@ -272,32 +277,39 @@ def duplicate_conference(publication):
ids
=
[]
db
=
current
.
globalenv
[
'db'
]
qcat
=
(
db
.
categories
.
code
==
'ACTI'
)
|
\
(
db
.
categories
.
code
==
'ACTN'
)
|
\
(
db
.
categories
.
code
==
'COM'
)
categories
=
db
.
categories
publications
=
db
.
publications
qcat
=
(
categories
.
code
==
'ACTI'
)
|
\
(
categories
.
code
==
'ACTN'
)
|
\
(
categories
.
code
==
'COM'
)
qmain
=
get_where_query
(
db
.
publications
)
qmain
=
get_where_query
(
publications
)
qmain
=
((
qmain
)
&
(
qcat
))
qmain
=
((
qmain
)
&
(
db
.
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
db
.
publications
.
title
==
publication
[
'title'
]))
qmain
=
((
qmain
)
&
(
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
publications
.
title
==
publication
[
'title'
]))
if
'id'
in
publication
and
publication
[
'id'
]:
qmain
=
((
qmain
)
&
(
db
.
publications
.
id
!=
publication
[
'id'
]))
qmain
=
((
qmain
)
&
(
publications
.
id
!=
publication
[
'id'
]))
# title, conference title, conference date and conference town
query
=
((
qmain
)
&
(
db
.
publications
.
conference_title
==
publication
[
'conference_title'
]))
query
=
((
query
)
&
(
db
.
publications
.
conference_dates
==
publication
[
'conference_dates'
]))
query
=
((
query
)
&
(
db
.
publications
.
conference_town
==
publication
[
'conference_town'
]))
qtitle
=
publications
.
conference_title
==
publication
[
'conference_title'
]
qdates
=
publications
.
conference_dates
==
publication
[
'conference_dates'
]
qtown
=
publications
.
conference_town
==
publication
[
'conference_town'
]
query
=
((
qmain
)
&
(
qtitle
))
query
=
((
query
)
&
(
qdates
))
query
=
((
query
)
&
(
qtown
))
extend_ids
(
db
,
query
,
ids
)
# title, conference date and conference town
query
=
((
query
)
&
(
db
.
publications
.
conference_dates
==
publication
[
'conference_dates'
]
))
query
=
((
query
)
&
(
db
.
publications
.
conference_town
==
publication
[
'conference_
town
'
]
))
query
=
((
query
)
&
(
qdates
))
query
=
((
query
)
&
(
q
town
))
extend_ids
(
db
,
query
,
ids
)
# title, conference title and conference town
query
=
((
qmain
)
&
(
db
.
publications
.
conference_title
==
publication
[
'conference_title'
]
))
query
=
((
query
)
&
(
db
.
publications
.
conference_town
==
publication
[
'conference_
town
'
]
))
query
=
((
qmain
)
&
(
qtitle
))
query
=
((
query
)
&
(
q
town
))
extend_ids
(
db
,
query
,
ids
)
return
ids
...
...
@@ -316,13 +328,15 @@ def duplicate_origin(publication):
ids
=
[]
db
=
current
.
globalenv
[
'db'
]
publications
=
db
.
publications
# protection against empty origin field
if
not
publication
[
'origin'
]:
return
ids
# look for publication with the same origin field
query
=
db
.
publications
.
id
!=
publication
[
'id'
]
query
=
((
query
)
&
(
db
.
publications
.
origin
==
publication
[
'origin'
]))
query
=
publications
.
id
!=
publication
[
'id'
]
query
=
((
query
)
&
(
publications
.
origin
==
publication
[
'origin'
]))
set_records
=
db
(
query
)
if
set_records
.
count
():
...
...
@@ -349,15 +363,15 @@ def duplicate_report(publication):
ids
=
[]
db
=
current
.
globalenv
[
'db'
]
qcat
=
db
.
categories
.
code
==
'AP'
publications
=
db
.
publications
qmain
=
get_where_query
(
db
.
publications
)
qmain
=
((
qmain
)
&
(
qcat
))
qmain
=
((
qmain
)
&
(
db
.
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
db
.
publications
.
title
==
publication
[
'title'
]))
qmain
=
get_where_query
(
publications
)
qmain
=
((
qmain
)
&
(
db
.
categories
.
code
==
'AP'
))
qmain
=
((
qmain
)
&
(
publications
.
id_teams
==
publication
[
'id_teams'
]))
qmain
=
((
qmain
)
&
(
publications
.
title
==
publication
[
'title'
]))
if
'id'
in
publication
and
publication
[
'id'
]:
qmain
=
((
qmain
)
&
(
db
.
publications
.
id
!=
publication
[
'id'
]))
qmain
=
((
qmain
)
&
(
publications
.
id
!=
publication
[
'id'
]))
extend_ids
(
db
,
qmain
,
ids
)
...
...
modules/countries.py
View file @
68597c28
modules/filters.py
View file @
68597c28
...
...
@@ -4,6 +4,7 @@
"""
def
CLEAN_COLLABORATION
(
value
):
"""Correct stupid mistakes on the collaboration field.
...
...
modules/harvest_tools/automaton.py
View file @
68597c28
...
...
@@ -159,7 +159,7 @@ class Automaton(object):
db
=
self
.
db
try
:
rec_id
=
db
.
publications
.
insert
(
**
fields
)
rec_id
=
db
.
publications
.
insert
(
**
fields
)
if
rec_id
:
return
1
...
...
modules/harvest_tools/base.py
View file @
68597c28
...
...
@@ -128,7 +128,7 @@ def learn_my_authors(db,
for
elem
in
diff
:
if
isinstance
(
elem
,
unicode
):
elem
=
elem
.
encode
(
'utf8'
)
family_name
=
elem
[
elem
.
rfind
(
'. '
)
+
2
:]
# extract family name
family_name
=
elem
[
elem
.
rfind
(
'. '
)
+
2
:]
if
family_name
not
in
row
.
authors
:
elems
.
append
(
elem
)
...
...
@@ -138,5 +138,5 @@ def learn_my_authors(db,
db
.
my_authors
[
row
.
id
]
=
dict
(
authors
=
', '
.
join
(
database_authors
))
class
ToolException
(
Exception
):
pass
class
ToolException
(
Exception
):
pass
modules/harvest_tools/msgcollection.py
View file @
68597c28
...
...
@@ -28,5 +28,3 @@ class MsgCollection(Storage):
"""
return
self
.
url
.
replace
(
"of=id"
,
"of=hb"
)
modules/invenio_tools/base.py
View file @
68597c28
...
...
@@ -66,9 +66,11 @@ def is_institute(record):
bool: true when the MARC record describes an institute
"""
# u'980': [{'b': [u'CK90', u'HEP200', u'PDGLIST', u'PPF', u'TOP500', u'WEB']},
# u'980': [
# {'b': [u'CK90', u'HEP200', u'PDGLIST', u'PPF', u'TOP500', u'WEB']},
# {'a': u'INSTITUTION'},
# {'a': u'CORE'}]}
# {'a': u'CORE'}
# ]
if
u
"980"
in
record
:
if
isinstance
(
record
[
u
"980"
],
list
):
...
...
@@ -77,8 +79,7 @@ def is_institute(record):
if
k
==
"a"
and
v
==
u
"INSTITUTION"
:
return
True
elif
isinstance
(
record
[
u
"980"
],
dict
)
and
\
"a"
in
record
[
u
"980"
]
and
\
elif
isinstance
(
record
[
u
"980"
],
dict
)
and
"a"
in
record
[
u
"980"
]
and
\
record
[
u
"980"
][
"a"
]
==
u
"INSTITUTION"
:
return
True
...
...
modules/invenio_tools/checkandfix.py
View file @
68597c28
...
...
@@ -35,21 +35,21 @@ _ref1 = r"(?P<p>[A-Za-z\. ]+) +(?P<v>\d+),? +(?P<c>[\d-]+) +\((?P<y>[\d]+)\)"
_ref2
=
r
"(?P<p>[A-Za-z\. ]+) +\((?P<y>\d+)\) +(?P<v>[\d]+):(?P<c>[\d-]+)"
DECODE_REF
=
[
re
.
compile
(
_ref1
),
re
.
compile
(
_ref2
)]
MONTHS
=
{
u
'Jan'
:
'01'
,
u
'Feb'
:
'02'
,
u
'Fev'
:
'02'
,
u
'Mar'
:
'03'
,
u
'Apr'
:
'04'
,
u
'Avr'
:
'04'
,
u
'May'
:
'05'
,
u
'Mai'
:
'05'
,
u
'Jun'
:
'06'
,
u
'Jul'
:
'07'
,
u
'Aug'
:
'08'
,
u
'Sep'
:
'09'
,
u
'Oct'
:
'10'
,
u
'Nov'
:
'11'
,
u
'Dec'
:
'12'
}
MONTHS
=
{
u
'Jan'
:
'01'
,
u
'Feb'
:
'02'
,
u
'Fev'
:
'02'
,
u
'Mar'
:
'03'
,
u
'Apr'
:
'04'
,
u
'Avr'
:
'04'
,
u
'May'
:
'05'
,
u
'Mai'
:
'05'
,
u
'Jun'
:
'06'
,
u
'Jul'
:
'07'
,
u
'Aug'
:
'08'
,
u
'Sep'
:
'09'
,
u
'Oct'
:
'10'
,
u
'Nov'
:
'11'
,
u
'Dec'
:
'12'
}
MSG_INVALID_HOST
=
"Invalid host"
...
...
@@ -79,7 +79,7 @@ REG_COLLABORATION = re.compile(regex.REG_COLLABORATION)
REG_CONF_DATES_1
=
re
.
compile
(
"(\d+) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})"
)
REG_CONF_DATES_2
=
\
re
.
compile
(
"(\d+) *([A-Z][a-z]{2}) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})"
)
re
.
compile
(
"(\d+) *([A-Z][a-z]{2}) *-? *(\d+) *([A-Z][a-z]{2}) *(\d{4})"
)
REG_CONF_DATES
=
re
.
compile
(
regex
.
REG_CONF_DATES
)
REG_SUBMITTED
=
re
.
compile
(
regex
.
REG_SUBMITTED
)
...
...
@@ -807,7 +807,7 @@ class CheckAndFix(object):
# 22 03 2011
m
=
DECODE_DD_MM_YYYY
.
match
(
dates
[
i
])
if
m
:
data
(
m
.
group
(
3
),
int
(
m
.
group
(
2
)),
int
(
m
.
group
(
1
)))
data
=
(
m
.
group
(
3
),
int
(
m
.
group
(
2
)),
int
(
m
.
group
(
1
)))
dates
[
i
]
=
'%s-%02i-%02i'
%
data
continue
...
...
modules/invenio_tools/exception.py
View file @
68597c28
...
...
@@ -2,6 +2,8 @@
""" invenio_tools.exception
"""
class
ExceptionUTF8
(
Exception
):
"""Exception in which unicode arguments are encoded as a string.
...
...
@@ -14,8 +16,21 @@ class ExceptionUTF8(Exception):
Exception
.
__init__
(
self
,
*
args
)
class
CdsException
(
ExceptionUTF8
):
pass
class
CheckException
(
ExceptionUTF8
):
pass
class
Marc12Exception
(
ExceptionUTF8
):
pass
class
RecordException
(
ExceptionUTF8
):
pass
class
XmlException
(
ExceptionUTF8
):
pass
class
CdsException
(
ExceptionUTF8
):
pass
class
CheckException
(
ExceptionUTF8
):
pass
class
Marc12Exception
(
ExceptionUTF8
):
pass
class
RecordException
(
ExceptionUTF8
):
pass
class
XmlException
(
ExceptionUTF8
):
pass
modules/invenio_tools/inveniostore.py
View file @
68597c28
...
...
@@ -435,11 +435,11 @@ class InvenioStore(object):
U{http://invenio-demo.cern.ch/help/hacking/search-engine-api}.
@rtype: unicode
@return: The format of the string (HTML, XML) depend on the
keyword C{of}.
For MARC12 format use C{xm}.
@return: The format of the string (HTML, XML) depend on the
keyword C{of}.
For MARC12 format use C{xm}.
@deprecated: the method L{get_ids} coupled with L{get_record} are
much more
efficient.
@deprecated: the method L{get_ids} coupled with L{get_record} are
much more
efficient.
"""
for
k
in
kwargs
:
...
...
modules/invenio_tools/iterrecord.py
View file @
68597c28
...
...
@@ -68,12 +68,22 @@ class IterRecord(object):
concatenate the following dictionary::
record[field] = [dict(subfield1=val1), dict(subfield2=val2), dict(subfield3=val3),...]
record[field] = [dict(subfield1=val1), dict(subfield2=val2, subfield3=val3),...]
record[field] = [
dict(subfield1=val1),
dict(subfield2=val2),
dict(subfield3=val3),...
]
record[field] = [
dict(subfield1=val1),
dict(subfield2=val2,
subfield3=val3),...
]
into a single one::
record[field] = dict1(subfield1=val1, subfield2=val2, subfield3=val3)
record[field] = dict1(subfield1=val1,
subfield2=val2,
subfield3=val3)
@type record: Record
@param record:
...
...
@@ -277,4 +287,3 @@ class IterRecord(object):
else
:
raise
StopIteration
()
modules/invenio_tools/record.py
View file @
68597c28
...
...
@@ -20,9 +20,9 @@ class Record(dict):
record[field] = [dict1(subfield1=..., subfield2=...),
dict2(subfield1=..., subfield2=...), ...]
In the MARC standard, the C{field} is a string containing at least three
digit
while the C{subfield} is a letter. The type of the C{field} is
unicode
and C{subfield} is string.
In the MARC standard, the C{field} is a string containing at least three
digit
while the C{subfield} is a letter. The type of the C{field} is
unicode
and C{subfield} is string.
The class comes with a collection of methods to extract the record
information masking the C{field} and the C{subfield} codification.
...
...
modules/invenio_tools/recordconf.py
View file @
68597c28
...
...
@@ -74,7 +74,8 @@ class RecordConf(RecordPubli):
"""
location
=
self
.
_get
(
u
"111"
,
"c"
)
# protection against [u'NOW 2012', u'Conca Specchiulla, Otranto, Lecce, Italy']
# protection against
# [u'NOW 2012', u'Conca Specchiulla, Otranto, Lecce, Italy']
if
isinstance
(
location
,
list
)
and
len
(
location
)
==
2
:
location
=
location
[
1
]
...
...
modules/invenio_tools/recordpubli.py
View file @
68597c28
...
...
@@ -228,7 +228,6 @@ class RecordPubli(Record):
if
"a"
in
di
:
authors
.
append
(
di
[
"a"
])
# remove duplicate entries and sort
authors
=
list
(
set
(
authors
))
if
cmpFct
:
...
...
@@ -452,7 +451,8 @@ class RecordPubli(Record):
published in a review.
"""
if
u
"773"
not
in
self
:
return
u
""
if
u
"773"
not
in
self
:
return
u
""
li
=
[]
for
k
in
(
"p"
,
"v"
,
"y"
,
"c"
):
...
...
modules/list_postprocessing.py
View file @
68597c28
...
...
@@ -132,4 +132,3 @@ def remove_undef(value, template, record):
"""
undef
=
current
.
T
(
plugin_dbui
.
UNDEF
)
return
re
.
sub
(
', *%s *,'
%
undef
,
','
,
value
)
\ No newline at end of file
modules/regex.py
View file @
68597c28
...
...
@@ -13,7 +13,8 @@ fields content.
# - CTA Consortium
# - any mixture of the above separated by a comma
#
REG_COLLABORATION
=
r
'^[A-Za-z0-9\-/, ]+([Cc]ollaboration|[Cc]onsortium|[Gg]roup)[s]?$'
REG_COLLABORATION
=
\
r
'^[A-Za-z0-9\-/, ]+([Cc]ollaboration|[Cc]onsortium|[Gg]roup)[s]?$'
# Valid Conference dates
# - 3 Dec 2012
...
...
modules/reporting_tools.py
View file @
68597c28
...
...
@@ -86,7 +86,9 @@ def get_sections(db, selector, row):
processing functions
"""
publications
=
db
.
publications
sections
=
[]
for
el
in
row
.
sections
.
split
(
','
):
# retrieve the section record
...
...
@@ -102,18 +104,18 @@ def get_sections(db, selector, row):
# query directive to extract publications for this section
# It includes foreign key constraints and user requirements
# related to team, project, authors and year
query
=
selector
.
query
(
db
.
publications
)
query
=
selector
.
query
(
publications
)
if
selector
.
year_start
and
not
selector
.
year_end
:
query
=
(
query
)
&
(
db
.
publications
.
year
==
selector
.
year_start
)
query
=
(
query
)
&
(
publications
.
year
==
selector
.
year_start
)
elif
selector
.
year_start
and
selector
.
year_end
:
q_start
=
db
.
publications
.
year
>=
selector
.
year_start
q_end
=
db
.
publications
.
year
<=
selector
.
year_end
q_start
=
publications
.
year
>=
selector
.
year_start
q_end
=
publications
.
year
<=
selector
.
year_end
query
=
(
query
)
&
((
q_start
)
&
(
q_end
))
if
selector
.
author
:
q_author
=
db
.
publications
.
authors_institute
.
contains
(
selector
.
author
)
q_author
=
publications
.
authors_institute
.
contains
(
selector
.
author
)
query
=
(
query
)
&
(
q_author
)
# add to the query the directive coming from the section itself
...
...
@@ -127,7 +129,7 @@ def get_sections(db, selector, row):
query
=
(
query
)
&
(
q_cat
)
if
section
.
conditions
:
q_conditions
=
smart_query
(
db
.
publications
,
section
.
conditions
)
q_conditions
=
smart_query
(
publications
,
section
.
conditions
)
query
=
(
query
)
&
(
q_conditions
)
section
.
query
=
query
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment