Commit efc20395 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Update controllers harvest and automaton to use 2 logging levels.

parent 6adcbb7e
......@@ -390,7 +390,8 @@ def run():
Search arguments are defined via the harvester selector.
"""
logger.debug(f"run harvester {request.vars.Harvester_selectorController}")
logger.info("-"*79)
logger.info(f"run harvester {request.vars.Harvester_selectorController}")
if db(db.affiliation_keys.id > 0).count() == 0:
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
......@@ -445,12 +446,14 @@ def run():
msg += "<hr/>"
return msg
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug("")
logger.debug(f"end of run harvester {selector.controller}:")
if logger.getEffectiveLevel() <= logging.INFO:
logger.info("")
logger.info(f"end of run harvester {selector.controller}:")
for el in collection_logs:
logger.debug(f" {el.title}: {el.found}")
logger.info(f" {el.title}: {el.found}")
logger.info("-"*79)
# delegate rendering to the report view
response.view = "harvest/layout.%s" % request.extension
......@@ -464,6 +467,8 @@ def run_all():
"""Run all harvesters in one go.
"""
logger.info(f"run all harvesters")
if db(db.affiliation_keys.id > 0).count() == 0:
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
......@@ -487,6 +492,9 @@ def run_all():
for harvester in harvesters:
logger.info("-"*79)
logger.info(f"run harvester {harvester.controller}")
tool = build_harvester_tool(
db,
harvester.id_teams,
......@@ -517,6 +525,17 @@ def run_all():
msg += "<hr/>"
return msg
if logger.getEffectiveLevel() <= logging.INFO:
logger.info("")
logger.info(f"end of run all harvesters:")
for el in collection_logs:
logger.info(f" {el.title}: {el.found}")
logger.info("-"*79)
logger.info("-"*79)
# tune selector parameters used in the report title
if query is None:
selector.id_projects = None
......
......@@ -288,8 +288,7 @@ class Automaton(object):
self.logs[-1].idle(MSG_IN_DB, publication.year)
logger = self.logger
logger.debug("")
logger.debug(f"{T2}record {rec_id} in db with id {publication.id}")
logger.info(f"{T2}record {rec_id} in db with id {publication.id}")
return publication.id
......@@ -506,7 +505,7 @@ class Automaton(object):
"""
logger = self.logger
logger.debug(f"process collection {collection}")
logger.info(f"process collection {collection}")
# alias
collection_logs = self.collection_logs
......@@ -528,7 +527,7 @@ class Automaton(object):
rec_ids = store.get_ids(**kwargs)
except CdsException as error:
logger.debug(f"exit process_collection: {error}")
logger.info(f"exit process_collection: {error}")
collection_logs[-1].url = store.last_search_url()
collection_logs[-1].error = error
return
......@@ -538,10 +537,11 @@ class Automaton(object):
collection_logs[-1].found = len(rec_ids)
if len(rec_ids) == 0:
logger.debug(f"no records found in {collection}")
logger.info(f"no records found in {collection}")
return
logger.debug(f"{len(rec_ids)} records found in {collection}")
logger.info(f"{len(rec_ids)} records found in {collection}")
logger.info("")
# remove form the list identifier already registered in the data base
# and log them
......@@ -564,7 +564,7 @@ class Automaton(object):
"""
logger = self.logger
logger.debug(f"{T4}process record {recjson['recid']} (process_recjson)")
logger.info(f"{T4}process record {recjson['recid']} (process_recjson)")
collection_logs = self.collection_logs
harvester = self.harvester
......@@ -584,7 +584,7 @@ class Automaton(object):
# check that the record is well formed
# repair non-conformity as far as possible
if not self.check_record(record):
logger.debug(f"{T4}{logs[-1].txt}")
logger.info(f"{T4}{logs[-1].txt}")
return
txt = ("(dry run)" if self.dry_run else "")
......@@ -593,11 +593,11 @@ class Automaton(object):
# insert the record in the database
self.insert_record(record)
if logger.getEffectiveLevel() == logging.DEBUG:
if logger.getEffectiveLevel() <= logging.INFO:
log = logs[-1]
action = log.action
action = (action.upper() if isinstance(action, str) else action)
logger.debug(f"{T4}log: {action} {log.txt}")
logger.info(f"{T4}log: {action} {log.txt}")
def process_recid(self, rec_id):
"""Process the publication identified by its record identifier:
......@@ -620,8 +620,8 @@ class Automaton(object):
"""
logger = self.logger
logger.debug("")
logger.debug(f"{T2}get record {rec_id} (process_recid)")
logger.info("")
logger.info(f"{T2}get record {rec_id} (process_recid)")
collection_logs = self.collection_logs
harvester = self.harvester
......@@ -660,8 +660,10 @@ class Automaton(object):
Collections are separated by a comma.
"""
self.logger.debug("")
self.logger.debug(f"process URL search -- {host} -- {collections}")
code = self.db.categories[self.id_category].code
self.logger.info("")
self.logger.info(f"process URL -- {host} -- {collections} -- {code}")
# extend harvester for logs
self.harvester.host = host
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment