Update controllers harvest and automaton to use 2 logging levels.

controllers/harvest.py

@@ -390,7 +390,8 @@ def run():
     Search arguments are defined via the harvester selector.
 
     """
-    logger.debug(f"run harvester {request.vars.Harvester_selectorController}")
+    logger.info("-"*79)
+    logger.info(f"run harvester {request.vars.Harvester_selectorController}")
 
     if db(db.affiliation_keys.id > 0).count() == 0:
         return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
@@ -445,12 +446,14 @@ def run():
         msg += "<hr/>"
         return msg
 
-    if logger.getEffectiveLevel() == logging.DEBUG:
-        logger.debug("")
-        logger.debug(f"end of run harvester {selector.controller}:")
+    if logger.getEffectiveLevel() <= logging.INFO:
+        logger.info("")
+        logger.info(f"end of run harvester {selector.controller}:")
 
         for el in collection_logs:
-            logger.debug(f"  {el.title}: {el.found}")
+            logger.info(f"  {el.title}: {el.found}")
+
+        logger.info("-"*79)
 
     # delegate rendering to the report view
     response.view = "harvest/layout.%s" % request.extension
@@ -464,6 +467,8 @@ def run_all():
     """Run all harvesters in one go.
 
     """
+    logger.info(f"run all harvesters")
+
     if db(db.affiliation_keys.id > 0).count() == 0:
         return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
 
@@ -487,6 +492,9 @@ def run_all():
 
         for harvester in harvesters:
 
+            logger.info("-"*79)
+            logger.info(f"run harvester {harvester.controller}")
+
             tool = build_harvester_tool(
                 db,
                 harvester.id_teams,
@@ -517,6 +525,17 @@ def run_all():
         msg += "<hr/>"
         return msg
 
+    if logger.getEffectiveLevel() <= logging.INFO:
+        logger.info("")
+        logger.info(f"end of run all harvesters:")
+
+        for el in collection_logs:
+            logger.info(f"  {el.title}: {el.found}")
+
+        logger.info("-"*79)
+
+    logger.info("-"*79)
+
     # tune selector parameters used in the report title
     if query is None:
         selector.id_projects = None

modules/harvest_tools/automaton.py

@@ -288,8 +288,7 @@ class Automaton(object):
         self.logs[-1].idle(MSG_IN_DB, publication.year)
 
         logger = self.logger
-        logger.debug("")
-        logger.debug(f"{T2}record {rec_id} in db with id {publication.id}")
+        logger.info(f"{T2}record {rec_id} in db with id {publication.id}")
 
         return publication.id
 
@@ -506,7 +505,7 @@ class Automaton(object):
 
         """
         logger = self.logger
-        logger.debug(f"process collection {collection}")
+        logger.info(f"process collection {collection}")
 
         # alias
         collection_logs = self.collection_logs
@@ -528,7 +527,7 @@ class Automaton(object):
             rec_ids = store.get_ids(**kwargs)
 
         except CdsException as error:
-            logger.debug(f"exit process_collection: {error}")
+            logger.info(f"exit process_collection: {error}")
             collection_logs[-1].url = store.last_search_url()
             collection_logs[-1].error = error
             return
@@ -538,10 +537,11 @@ class Automaton(object):
         collection_logs[-1].found = len(rec_ids)
 
         if len(rec_ids) == 0:
-            logger.debug(f"no records found in {collection}")
+            logger.info(f"no records found in {collection}")
             return
 
-        logger.debug(f"{len(rec_ids)} records found in {collection}")
+        logger.info(f"{len(rec_ids)} records found in {collection}")
+        logger.info("")
 
         # remove form the list identifier already registered in the data base
         # and log them
@@ -564,7 +564,7 @@ class Automaton(object):
 
         """
         logger = self.logger
-        logger.debug(f"{T4}process record {recjson['recid']} (process_recjson)")
+        logger.info(f"{T4}process record {recjson['recid']} (process_recjson)")
 
         collection_logs = self.collection_logs
         harvester = self.harvester
@@ -584,7 +584,7 @@ class Automaton(object):
         # check that the record is well formed
         # repair non-conformity as far as possible
         if not self.check_record(record):
-            logger.debug(f"{T4}{logs[-1].txt}")
+            logger.info(f"{T4}{logs[-1].txt}")
             return
 
         txt = ("(dry run)" if self.dry_run else "")
@@ -593,11 +593,11 @@ class Automaton(object):
         # insert the record in the database
         self.insert_record(record)
 
-        if logger.getEffectiveLevel() == logging.DEBUG:
+        if logger.getEffectiveLevel() <= logging.INFO:
             log = logs[-1]
             action = log.action
             action = (action.upper() if isinstance(action, str) else action)
-            logger.debug(f"{T4}log: {action} {log.txt}")
+            logger.info(f"{T4}log: {action} {log.txt}")
 
     def process_recid(self, rec_id):
         """Process the publication identified by its record identifier:
@@ -620,8 +620,8 @@ class Automaton(object):
 
         """
         logger = self.logger
-        logger.debug("")
-        logger.debug(f"{T2}get record {rec_id} (process_recid)")
+        logger.info("")
+        logger.info(f"{T2}get record {rec_id} (process_recid)")
 
         collection_logs = self.collection_logs
         harvester = self.harvester
@@ -660,8 +660,10 @@ class Automaton(object):
                 Collections are separated by a comma.
 
         """
-        self.logger.debug("")
-        self.logger.debug(f"process URL search -- {host} -- {collections}")
+        code = self.db.categories[self.id_category].code
+
+        self.logger.info("")
+        self.logger.info(f"process URL -- {host} -- {collections} -- {code}")
 
         # extend harvester for logs
         self.harvester.host = host