Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
limbra
limbra
Commits
efc20395
Commit
efc20395
authored
Feb 06, 2020
by
LE GAC Renaud
Browse files
Update controllers harvest and automaton to use 2 logging levels.
parent
6adcbb7e
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
40 additions
and
19 deletions
+40
-19
controllers/harvest.py
controllers/harvest.py
+24
-5
modules/harvest_tools/automaton.py
modules/harvest_tools/automaton.py
+16
-14
No files found.
controllers/harvest.py
View file @
efc20395
...
...
@@ -390,7 +390,8 @@ def run():
Search arguments are defined via the harvester selector.
"""
logger
.
debug
(
f
"run harvester
{
request
.
vars
.
Harvester_selectorController
}
"
)
logger
.
info
(
"-"
*
79
)
logger
.
info
(
f
"run harvester
{
request
.
vars
.
Harvester_selectorController
}
"
)
if
db
(
db
.
affiliation_keys
.
id
>
0
).
count
()
==
0
:
return
inline_alert
(
T
(
"Error"
),
T
(
MSG_NO_AFFILIATION
))
...
...
@@ -445,12 +446,14 @@ def run():
msg
+=
"<hr/>"
return
msg
if
logger
.
getEffectiveLevel
()
=
=
logging
.
DEBUG
:
logger
.
debug
(
""
)
logger
.
debug
(
f
"end of run harvester
{
selector
.
controller
}
:"
)
if
logger
.
getEffectiveLevel
()
<
=
logging
.
INFO
:
logger
.
info
(
""
)
logger
.
info
(
f
"end of run harvester
{
selector
.
controller
}
:"
)
for
el
in
collection_logs
:
logger
.
debug
(
f
"
{
el
.
title
}
:
{
el
.
found
}
"
)
logger
.
info
(
f
"
{
el
.
title
}
:
{
el
.
found
}
"
)
logger
.
info
(
"-"
*
79
)
# delegate rendering to the report view
response
.
view
=
"harvest/layout.%s"
%
request
.
extension
...
...
@@ -464,6 +467,8 @@ def run_all():
"""Run all harvesters in one go.
"""
logger
.
info
(
f
"run all harvesters"
)
if
db
(
db
.
affiliation_keys
.
id
>
0
).
count
()
==
0
:
return
inline_alert
(
T
(
"Error"
),
T
(
MSG_NO_AFFILIATION
))
...
...
@@ -487,6 +492,9 @@ def run_all():
for
harvester
in
harvesters
:
logger
.
info
(
"-"
*
79
)
logger
.
info
(
f
"run harvester
{
harvester
.
controller
}
"
)
tool
=
build_harvester_tool
(
db
,
harvester
.
id_teams
,
...
...
@@ -517,6 +525,17 @@ def run_all():
msg
+=
"<hr/>"
return
msg
if
logger
.
getEffectiveLevel
()
<=
logging
.
INFO
:
logger
.
info
(
""
)
logger
.
info
(
f
"end of run all harvesters:"
)
for
el
in
collection_logs
:
logger
.
info
(
f
"
{
el
.
title
}
:
{
el
.
found
}
"
)
logger
.
info
(
"-"
*
79
)
logger
.
info
(
"-"
*
79
)
# tune selector parameters used in the report title
if
query
is
None
:
selector
.
id_projects
=
None
...
...
modules/harvest_tools/automaton.py
View file @
efc20395
...
...
@@ -288,8 +288,7 @@ class Automaton(object):
self
.
logs
[
-
1
].
idle
(
MSG_IN_DB
,
publication
.
year
)
logger
=
self
.
logger
logger
.
debug
(
""
)
logger
.
debug
(
f
"
{
T2
}
record
{
rec_id
}
in db with id
{
publication
.
id
}
"
)
logger
.
info
(
f
"
{
T2
}
record
{
rec_id
}
in db with id
{
publication
.
id
}
"
)
return
publication
.
id
...
...
@@ -506,7 +505,7 @@ class Automaton(object):
"""
logger
=
self
.
logger
logger
.
debug
(
f
"process collection
{
collection
}
"
)
logger
.
info
(
f
"process collection
{
collection
}
"
)
# alias
collection_logs
=
self
.
collection_logs
...
...
@@ -528,7 +527,7 @@ class Automaton(object):
rec_ids
=
store
.
get_ids
(
**
kwargs
)
except
CdsException
as
error
:
logger
.
debug
(
f
"exit process_collection:
{
error
}
"
)
logger
.
info
(
f
"exit process_collection:
{
error
}
"
)
collection_logs
[
-
1
].
url
=
store
.
last_search_url
()
collection_logs
[
-
1
].
error
=
error
return
...
...
@@ -538,10 +537,11 @@ class Automaton(object):
collection_logs
[
-
1
].
found
=
len
(
rec_ids
)
if
len
(
rec_ids
)
==
0
:
logger
.
debug
(
f
"no records found in
{
collection
}
"
)
logger
.
info
(
f
"no records found in
{
collection
}
"
)
return
logger
.
debug
(
f
"
{
len
(
rec_ids
)
}
records found in
{
collection
}
"
)
logger
.
info
(
f
"
{
len
(
rec_ids
)
}
records found in
{
collection
}
"
)
logger
.
info
(
""
)
# remove form the list identifier already registered in the data base
# and log them
...
...
@@ -564,7 +564,7 @@ class Automaton(object):
"""
logger
=
self
.
logger
logger
.
debug
(
f
"
{
T4
}
process record
{
recjson
[
'recid'
]
}
(process_recjson)"
)
logger
.
info
(
f
"
{
T4
}
process record
{
recjson
[
'recid'
]
}
(process_recjson)"
)
collection_logs
=
self
.
collection_logs
harvester
=
self
.
harvester
...
...
@@ -584,7 +584,7 @@ class Automaton(object):
# check that the record is well formed
# repair non-conformity as far as possible
if
not
self
.
check_record
(
record
):
logger
.
debug
(
f
"
{
T4
}{
logs
[
-
1
].
txt
}
"
)
logger
.
info
(
f
"
{
T4
}{
logs
[
-
1
].
txt
}
"
)
return
txt
=
(
"(dry run)"
if
self
.
dry_run
else
""
)
...
...
@@ -593,11 +593,11 @@ class Automaton(object):
# insert the record in the database
self
.
insert_record
(
record
)
if
logger
.
getEffectiveLevel
()
=
=
logging
.
DEBUG
:
if
logger
.
getEffectiveLevel
()
<
=
logging
.
INFO
:
log
=
logs
[
-
1
]
action
=
log
.
action
action
=
(
action
.
upper
()
if
isinstance
(
action
,
str
)
else
action
)
logger
.
debug
(
f
"
{
T4
}
log:
{
action
}
{
log
.
txt
}
"
)
logger
.
info
(
f
"
{
T4
}
log:
{
action
}
{
log
.
txt
}
"
)
def
process_recid
(
self
,
rec_id
):
"""Process the publication identified by its record identifier:
...
...
@@ -620,8 +620,8 @@ class Automaton(object):
"""
logger
=
self
.
logger
logger
.
debug
(
""
)
logger
.
debug
(
f
"
{
T2
}
get record
{
rec_id
}
(process_recid)"
)
logger
.
info
(
""
)
logger
.
info
(
f
"
{
T2
}
get record
{
rec_id
}
(process_recid)"
)
collection_logs
=
self
.
collection_logs
harvester
=
self
.
harvester
...
...
@@ -660,8 +660,10 @@ class Automaton(object):
Collections are separated by a comma.
"""
self
.
logger
.
debug
(
""
)
self
.
logger
.
debug
(
f
"process URL search --
{
host
}
--
{
collections
}
"
)
code
=
self
.
db
.
categories
[
self
.
id_category
].
code
self
.
logger
.
info
(
""
)
self
.
logger
.
info
(
f
"process URL --
{
host
}
--
{
collections
}
--
{
code
}
"
)
# extend harvester for logs
self
.
harvester
.
host
=
host
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment