Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
limbra
limbra
Commits
29725268
Commit
29725268
authored
Jun 29, 2017
by
LE GAC Renaud
Browse files
Migrate Talks.
parent
927ce661
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
91 additions
and
16 deletions
+91
-16
modules/harvest_tools/checkandfix.py
modules/harvest_tools/checkandfix.py
+4
-1
modules/harvest_tools/talks.py
modules/harvest_tools/talks.py
+11
-8
modules/invenio_tools/factory.py
modules/invenio_tools/factory.py
+7
-7
tests/basis/test_11_CheckAndFix_proceeding.py
tests/basis/test_11_CheckAndFix_proceeding.py
+8
-0
tests/basis/test_12_CheckAndFix_thesis.py
tests/basis/test_12_CheckAndFix_thesis.py
+0
-0
tests/scan/test_Talk.py
tests/scan/test_Talk.py
+61
-0
No files found.
modules/harvest_tools/checkandfix.py
View file @
29725268
...
...
@@ -658,7 +658,7 @@ class CheckAndFix(object):
record
.
my_authors
=
value
def
is_conference
(
self
,
record
):
"""Check that the record
described a
conference ta
lk / proceeding
.
"""Check that the record
contains
conference
da
ta.
Args:
record (RecordPubli):
...
...
@@ -675,6 +675,9 @@ class CheckAndFix(object):
if
not
isinstance
(
record
,
RecordConf
):
raise
CheckException
(
MSG_NO_CONF
)
if
u
"meeting_name"
not
in
record
:
raise
CheckException
(
MSG_NO_CONF
)
def
is_thesis
(
self
,
record
):
"""Check that the record described a thesis.
...
...
modules/harvest_tools/talks.py
View file @
29725268
...
...
@@ -18,10 +18,12 @@ class Talks(Automaton):
"""Check the content of the talk in order to fix non conformities.
Args:
record (RecordConf): record describing a conference.
record (RecordConf):
record describing a conference.
Returns:
bool: ``False`` when a non conformity is found and can not be
bool:
``False`` when a non conformity is found and can not be
corrected.
"""
...
...
@@ -34,10 +36,9 @@ class Talks(Automaton):
try
:
self
.
check
.
is_conference
(
record
)
self
.
check
.
country
(
record
)
self
.
check
.
conference_date
(
record
,
self
.
harvester
.
host
)
self
.
check
.
conference_date
(
record
)
self
.
check
.
submitted
(
record
)
self
.
check
.
year
(
record
)
self
.
check
.
format_authors
(
record
,
fmt
=
"F. Last"
)
self
.
check
.
get_my_authors
(
record
,
sort
=
True
)
...
...
@@ -57,23 +58,25 @@ class Talks(Automaton):
"""Insert a conference talk in the database.
Args:
record (RecordConf): record describing a conference.
record (RecordConf):
record describing a conference.
Returns:
int: one when the record is inserted / updated in the database
int:
one when the record is inserted / updated in the database
zero otherwise.
"""
# alias
oai_url
=
record
.
oai_url
()
year
=
record
.
year
()
year
=
record
.
conference_
year
()
# alias for the conference information
conference_dates
=
record
.
conference_dates
()
conference_title
=
record
.
conference_title
()
first_author
=
record
.
first_author
()
id_country
=
self
.
search_country
(
record
.
conference_country
())
submitted
=
record
.
submitted
()
[
0
]
submitted
=
record
.
submitted
()
title
=
record
.
title
()
# get the collaboration identifier
...
...
modules/invenio_tools/factory.py
View file @
29725268
...
...
@@ -69,17 +69,17 @@ def add_conference_data(recjson):
+---------------+-----------------------------------------------+
| meeting_name | closing_date, coference_code, country, date, |
| | location, opening_date, year |
| meeting
| recid, url |
| meeting
_note
| recid, url |
+---------------+-----------------------------------------------+
Args:
recjson (dict): record data (MarcJSON)
Rais
e:
CdsException:
-
no conference
identifier and
key in the recjson
- conference recjson found but with a wrong identifier
- conference not found
Not
e:
* Fields are not added when there is no conference identifier and
no conference key in the recjson
.
* The method CheckAndFix.is_conference will identify that case.
"""
# ........................................................................
#
...
...
@@ -107,7 +107,7 @@ def add_conference_data(recjson):
break
if
conf_id
is
None
and
conf_key
is
None
:
r
aise
CdsException
(
MSG_NO_CONF_ID_KEY
)
r
eturn
# ........................................................................
#
...
...
tests/basis/test_11_CheckAndFix_proceeding.py
View file @
29725268
...
...
@@ -20,6 +20,7 @@
"""
import
pytest
from
harvest_tools.checkandfix
import
CheckAndFix
from
harvest_tools.exception
import
CheckException
from
invenio_tools
import
load_record
...
...
@@ -42,6 +43,13 @@ def test_is_conference(svc, reccds, recins):
assert
svc
.
is_conference
(
reccds
)
is
None
assert
svc
.
is_conference
(
recins
)
is
None
# test exception
# the publication cds2242595 is a talk without conference data
#
reccds2
=
load_record
(
"cds.cern.ch"
,
2242595
)
with
pytest
.
raises
(
CheckException
):
svc
.
is_conference
(
reccds2
)
def
test_country
(
svc
,
reccds
,
recins
):
assert
svc
.
country
(
reccds
)
is
None
...
...
tests/basis/test_12_CheckAndFix_t
alk
.py
→
tests/basis/test_12_CheckAndFix_t
hesis
.py
View file @
29725268
File moved
tests/scan/test_Talk.py
0 → 100644
View file @
29725268
# -*- coding: utf-8 -*-
"""test_Talk
* Harvester is Talks
* Store is cds.cern.ch
* LHCb COM for the current year
* Check that all error messages are expected
"""
import
pytest
from
gluon
import
current
from
harvest_tools.talks
import
Talks
from
harvest_tools.factory
import
build_harvester_tool
from
test_tools
import
messages
@
pytest
.
fixture
(
scope
=
"module"
)
def
harvester_messages
():
return
messages
()
def
test_lhcb_com
(
harvester_messages
):
"""Harvest LHCb talk for a given year.
This test is useful to:
* debug an harvester
* profile its performance to see where the time is spent.
* compare different implementation to measure improvements.
* ...
"""
# These parameter only make sense if you are inserting record in database
# Select the current year in order to test different case
db
=
current
.
db
id_team
=
7
# LHCb
id_project
=
8
# LHCb
id_category
=
9
# COM
year
=
current
.
request
.
now
.
year
# build the harvester
tool
=
build_harvester_tool
(
db
,
id_team
,
id_project
,
"talks"
,
id_category
,
year_start
=
str
(
year
),
year_end
=
""
,
dry_run
=
True
,
debug
=
True
)
assert
isinstance
(
tool
,
Talks
)
# run the harvester
tool
.
process_url
(
"cds.cern.ch"
,
"LHCb Talks"
)
# analyse the log
# Number of talk cannot be check since it evolve within a year
# Only test that there are no unexpected messages
msgs
=
set
([
el
.
txt
for
el
in
tool
.
logs
])
assert
msgs
.
issubset
(
harvester_messages
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment