automaton.py 21.6 KB
Newer Older
1
""" harvest_tools.automaton
2 3 4 5 6

"""
import re
import traceback

7

8 9 10 11
from .base import (MSG_FIX_ORIGIN,
                   MSG_IN_DB,
                   ToolException)
from .checkandfix import CheckAndFix
12
from gluon.storage import Storage
LE GAC Renaud's avatar
LE GAC Renaud committed
13 14
from invenio_tools import (CdsException,
                           InvenioStore,
15
                           OAI_URL)
LE GAC Renaud's avatar
LE GAC Renaud committed
16
from invenio_tools.factory import build_record
17 18
from .msg import Msg
from .msgcollection import MsgCollection
19
from plugin_dbui import CALLBACK_ERRORS, get_id
20

21

22 23 24
MSG_NO_CAT = 'Select a "category" !!!'
MSG_NO_PROJECT = 'Select a "project" !!!'
MSG_NO_TEAM = 'Select a "team" !!!'
25

LE GAC Renaud's avatar
LE GAC Renaud committed
26
MSG_INSERT_FAIL = "Fail to insert the new record in the database."
27

28 29 30 31
# search collection when using inspirehep
# require for "Hal Hidden"
REG_COLLECTION = re.compile(r"cc([A-Za-z ]+)(and|$)")

32

33
class Automaton(object):
34
    """Base class to search and process publications:
35

36
        * Decode the selector defining user criteria.
LE GAC Renaud's avatar
LE GAC Renaud committed
37
        * Search in the store publications matching user criteria.
LE GAC Renaud's avatar
LE GAC Renaud committed
38
        * Instantiate the record and check it.
39
        * Insert new records in the database.
40

41 42
    Note:
        The parameters of the search are defined by the current ``request``.
43

44 45 46
    The logic implements in the ``Automaton`` class is the following:

        #. Ask to the store, all the `record_id` satisfying the user request.
LE GAC Renaud's avatar
LE GAC Renaud committed
47 48
        #. Reject `record_id` contains in the *origin* field of a
           database entry.
LE GAC Renaud's avatar
LE GAC Renaud committed
49
        #. Request to the store, the JSON description of the publications
LE GAC Renaud's avatar
LE GAC Renaud committed
50 51 52 53
           and decode them.
        #. Reject the record for which the *secondary_oai_url* is contained in
           the *origin* field of a database entry. Update the *origin* field
           of the database record.
54
        #. Check that the *oai* of the publication is defined and well formed.
LE GAC Renaud's avatar
LE GAC Renaud committed
55 56
           Recover it, if it is not the case. At this stage the OAI is always
           defined.
57 58
        #. Reject temporarily publication.
        #. Check that *authors* are defined.
59
           Reject the publication if it is not the case.
60
        #. Check that *my institute* is in the list of the institutes
61 62 63 64 65 66
           signing the publication. Reject the publication if it is
           not the case. When the affiliation are not defined,
           try to recover this case, by finding the author of my institute
           signing the publication. This recovery procedure uses
           the *author rescue list*. Reject the record when the recovery
           procedure failed.
67
        #. Check that the *collaboration*, if defined, is well formed.
68
           Reject the publication if it is not the case
69 70 71 72 73
        #. Several check are applied depending on the publication type.
        #. At the end of this process, the publisher, the authors are
           formatted and the list of signatories of my institute extracted.

    Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
        db (gluon.DAL):
            the database connection.

        id_team (int):
            the identifier of the team in the database.

        id_project (int):
            the identifier of the project in the database.

        automaton (unicode):
            the name of the automaton which will be used to process the data.
            Possible values are: ``articles``, ``notes``, ``preprints``,
            ``proceedings``, ``reports``, ``talks`` and ``theses``.

        id_category (int):
            the identifier of the category of publication

        year_start (int):
            starting year for the scan

        year_end (int):
            ending year of the scan

        dry_run (bool):
            new records are not inserted in the database when ``True``.

        debug (bool):
            activate the verbose mode when ``True``.
102 103

    Raises:
LE GAC Renaud's avatar
LE GAC Renaud committed
104 105
        ToolException:
            * team or project or the publication category not defined
106

107 108 109 110 111
    """
    def __init__(self,
                 db,
                 id_team,
                 id_project,
112
                 automaton,
113 114 115 116 117 118 119
                 id_category,
                 year_start=None,
                 year_end=None,
                 dry_run=True,
                 debug=False):

        # protection team, project and/or category have to be defined
LE GAC Renaud's avatar
LE GAC Renaud committed
120
        if not id_team:
121 122
            raise ToolException(MSG_NO_TEAM)

LE GAC Renaud's avatar
LE GAC Renaud committed
123
        if not id_project:
124 125
            raise ToolException(MSG_NO_PROJECT)

LE GAC Renaud's avatar
LE GAC Renaud committed
126
        if not id_category:
127 128
            raise ToolException(MSG_NO_CAT)

129
        self.check = CheckAndFix(debug)
LE GAC Renaud's avatar
LE GAC Renaud committed
130 131 132 133 134 135 136 137 138 139 140 141 142
        self.collection_logs = []
        self.controller = automaton
        self.db = db
        self.dbg = debug
        self.dry_run = dry_run
        self.id_category = id_category
        self.id_team = id_team
        self.id_project = id_project
        self.logs = []
        self.store = None
        self.year_start = year_start
        self.year_end = year_end

143
        # Construct harvester Storage needed for the log
LE GAC Renaud's avatar
LE GAC Renaud committed
144 145 146 147
        self.harvester = Storage(id_teams=id_team,
                                 id_projects=id_project,
                                 controller=automaton,
                                 id_categories=id_category)
148

149 150 151 152 153
        # Identifier of the categories preprint and articles
        # Used by the method _is_record_in_db
        self._id_preprint = get_id(db.categories, code="PRE")
        self._id_article = get_id(db.categories, code="ACL")

154 155 156
    def _insert_in_db(self, log_year="", **fields):
        """Insert the record in the database, handling database exception.

157
        Args:
158
            log_year (str): year of the record for the log
159

160
        Keyword Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
161 162
            **fields:
                keyword arguments defining the record values to be
163
                inserted in the database.
164

165
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
166 167
            int:
                one when the record is inserted / updated in the database,
168
                zero otherwise.
169 170 171 172 173

        """
        db = self.db

        try:
LE GAC Renaud's avatar
LE GAC Renaud committed
174
            rec_id = db.publications.insert(**fields)
LE GAC Renaud's avatar
LE GAC Renaud committed
175 176
            if rec_id:
                return 1
177

LE GAC Renaud's avatar
LE GAC Renaud committed
178
            # operation can be reject by callback table._before_insert
LE GAC Renaud's avatar
LE GAC Renaud committed
179
            else:
LE GAC Renaud's avatar
LE GAC Renaud committed
180
                msg = MSG_INSERT_FAIL
LE GAC Renaud's avatar
LE GAC Renaud committed
181 182
                if CALLBACK_ERRORS in db.publications:
                    msg = db.publications._callback_errors
183

LE GAC Renaud's avatar
LE GAC Renaud committed
184 185 186
                # reduce the error message
                if isinstance(msg, list):
                    msg = "%s %s" % (msg[0], msg[-1])
187

LE GAC Renaud's avatar
LE GAC Renaud committed
188 189
                self.logs[-1].reject(msg, log_year)
                return 0
190

LE GAC Renaud's avatar
LE GAC Renaud committed
191 192
        # operation can be rejected by the database
        except Exception as dbe:
193
            self.logs[-1].reject(str(dbe), log_year)
LE GAC Renaud's avatar
LE GAC Renaud committed
194
            return 0
195

LE GAC Renaud's avatar
LE GAC Renaud committed
196 197 198 199 200 201 202
    def _is_record_in_db(self,
                         collection_title,
                         host=None,
                         rec_id=None,
                         oai_url=None):
        """Return the database identifier when the publication is registered.
        The search is based on the ``origin`` field and on the primary OAI.
203

204 205
        Note:
            A new log entry is created when a record is found.
206

207
        Args:
208
            title (str): the title of the publication.
209 210

        Keyword Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
211 212
            host (unicode):
                the store. possible values are ``cds.cern.ch`` or
213 214
                ``inspirehep.net``. To be used with *rec_id*.

LE GAC Renaud's avatar
LE GAC Renaud committed
215 216 217 218 219 220
            rec_id (int):
                the record identifier in the store

            oai_url (unicode):
                the URL of the record in the store.
                Either use *host* and *rec_id* or *oai_url*
221

222
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
223 224
            int:
                the id of the record in the database when a record is found,
225
                0 otherwise.
226

227
        Raises:
LE GAC Renaud's avatar
LE GAC Renaud committed
228 229
            ValueError:
                * keyword arguments are not defined properly.
230

231 232
        """
        db = self.db
233
        harvester = self.harvester
234

235 236 237 238 239 240 241 242
        # build the OAI URL
        if host is not None and rec_id is not None and oai_url is None:
            url = OAI_URL % (host, rec_id)
        elif host is None and rec_id is None and oai_url is not None:
            url = oai_url
        else:
            raise ValueError

LE GAC Renaud's avatar
LE GAC Renaud committed
243
        # protection empty URL
244 245 246
        if len(url) == 0:
            return 0

247 248 249
        # check the OAI
        query = db.publications.origin.contains(url)
        setrows = db(query)
250

251
        if setrows.count() == 0:
252
            return 0
253

254
        # one record found
255 256
        columns = [db.publications.id,
                   db.publications.id_categories,
257 258 259
                   db.publications.title,
                   db.publications.year]
        publication = setrows.select(*columns).first()
260

261 262
        # Note:
        # The category for the publication and the harvester have to be equal.
263 264 265 266 267 268 269
        # However, keep the record if it is a preprint when the harvester
        # looks for articles. This is required to transform a preprint
        # into article
        #
        # Category can disagree when the publication is an article and
        # the harvester look for preprint. In that case, keep the article
        #
270
        if publication.id_categories != harvester.id_categories:
271 272 273 274 275 276 277

            is_preprint_to_article = \
                publication.id_categories == self._id_preprint \
                and harvester.id_categories == self._id_article

            if is_preprint_to_article:
                return 0
278 279

        # log
280
        self.logs.append(Msg(harvester=harvester,
LE GAC Renaud's avatar
LE GAC Renaud committed
281
                             collection=collection_title,
282 283 284 285 286
                             record_id=rec_id,
                             title=publication.title))

        self.logs[-1].idle(MSG_IN_DB, publication.year)

LE GAC Renaud's avatar
LE GAC Renaud committed
287 288 289
        if self.dbg:
            print("\trecord already in db:", rec_id, "->", publication.id)

290
        return publication.id
291

292 293 294 295 296
    def _search_parameters(self, collection):
        """Build the keywords to steer the URL search in invenio store.
        The main parameter is the collection and the date range defined
        in the selector.

297
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
298 299 300
            collection (unicode):
                string defining the collection in the store.
                The syntax depends on the invenio store:
301 302 303

                    * ``"find cn d0 and tc p and not tc c"``
                    * ``"LHCb Papers"``.
304

305
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
306 307 308
            dict:
                the key are a sub-set of those defined in
                :meth:`invenio_tools.InvenioStore.get_ids`.
309 310

        """
LE GAC Renaud's avatar
LE GAC Renaud committed
311 312
        year_start = self.year_start
        year_end = self.year_end
313 314

        # INSPIREHEP store
LE GAC Renaud's avatar
LE GAC Renaud committed
315
        if collection.startswith("find"):
316 317 318

            query = collection

LE GAC Renaud's avatar
LE GAC Renaud committed
319 320
            if year_start and not year_end:
                query += " and date %s" % year_start
321

LE GAC Renaud's avatar
LE GAC Renaud committed
322 323
            elif not year_start and year_end:
                query += " and date %s" % year_end
324

LE GAC Renaud's avatar
LE GAC Renaud committed
325
            elif year_start and year_end:
326
                query += " and date > %s and date < %s " \
LE GAC Renaud's avatar
LE GAC Renaud committed
327
                         % (year_start - 1, year_end + 1)
328 329 330

            dic = dict(p=query,  # query à la spires
                       rg=1000,  # maximum number of records returned
LE GAC Renaud's avatar
LE GAC Renaud committed
331 332
                       sf="year",  # sort by date
                       so="d")  # descending order
333

334 335 336 337 338 339 340 341 342
            # handle the cc keyword (true inspirehep collection)
            match = REG_COLLECTION.search(query)
            if match:
                dic["cc"] = match.group(1).strip()
                dic["p"] = REG_COLLECTION.sub("", query).strip()
                dic["p"] = dic["p"].replace("  ", " ")
                if dic["p"] == "find":
                    del dic["p"]

343 344 345
        # CERN INVENIO store
        else:

LE GAC Renaud's avatar
LE GAC Renaud committed
346 347
            if year_start and not year_end:
                rex = year_start
348

LE GAC Renaud's avatar
LE GAC Renaud committed
349 350
            elif not year_start and year_end:
                rex = year_end
351

LE GAC Renaud's avatar
LE GAC Renaud committed
352 353
            elif year_start and year_end:
                li = [str(el) for el in xrange(year_start, year_end + 1)]
LE GAC Renaud's avatar
LE GAC Renaud committed
354
                rex = "|".join(li)
355 356

            dic = dict(cc=collection,  # collection
LE GAC Renaud's avatar
LE GAC Renaud committed
357 358
                       f1="year",  # search on year
                       m1="r",  # use regular expression
359
                       p1=rex,  # regular expression defining year
LE GAC Renaud's avatar
LE GAC Renaud committed
360 361
                       sf="year",  # sort by date
                       so="d")  # descending order
362 363
        return dic

LE GAC Renaud's avatar
LE GAC Renaud committed
364
    def check_record(self, record):
365 366
        """Check the content of the record in order to fix non-conformities.
        Return ``False`` when non-conformities are found and can not be
367 368
        corrected.

369 370 371
        Note:
            Some checks depend on the type of publications and have to be
            implemented in inherited class.
372

373
        Note:
LE GAC Renaud's avatar
LE GAC Renaud committed
374
            The order of the checks matter. It should be OAI,
375 376
            temporary record, authors, my authors and then a series of checks
            specific to the publication type.
377

378
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
379 380
            record (Record):
                JSON record describing the publication.
381

382
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
383 384
            bool:
                ``False`` when a non-conformity is found and can not be
385
                corrected.
386 387 388

        """
        if self.dbg:
389
            print("check record")
390 391

        try:
392
            self.check.is_oai(record)
393

394
            if self.check.is_bad_oai_used(record):
LE GAC Renaud's avatar
LE GAC Renaud committed
395
                self.logs[-1].idle(MSG_IN_DB, record.submitted())
396 397
                return False

398 399
            self.check.temporary_record(record)
            self.check.authors(record)
400
            self.check.my_affiliation(record, self.id_project, self.id_team)
401 402 403
            self.check.collaboration(record)

        except Exception as e:
404
            self.logs[-1].reject(e, record=record)
405 406 407 408
            return False

        return True

409
    def get_record_by_fields(self, oai_url, year, **kwargs):
410 411
        """Get database record matching fields values defined
        in the keyword arguments.
412

413
        Note:
414 415
            This method is required to deal with publication entered by hand
            and found later by an harvester.
416

417
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
418 419 420 421
            oai_url (unicode):
                the oai_url, *e.g.* ``http://cds.cern.ch/record/123456``.
                The origin field of the existing database record is update to
                **oai_url** when a match is found.
422

LE GAC Renaud's avatar
LE GAC Renaud committed
423 424
            year (int):
                the year of the publication. It is used
425 426 427
                by the search algorithm and by the logger.

        Keyword Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
428 429 430
            kwargs (unicode):
                 a series of key, value pair where the key is the name of a
                 publications database field.
431

432
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
433 434 435 436 437
            tuple:
                ``(id, status)`` which contains the ``id`` of the record.
                The ``id`` is equal to ``None`` when there is no matching.
                The ``status`` is equal to one when the existing record was
                modified zero otherwise.
438 439 440

        """
        if self.dbg:
441
            print("get existing record by fields")
442

443
        # alias
444
        db = self.db
445
        logs = self.logs
446

447 448 449
        # add the publication year to search criteria
        if year:
            kwargs["year"] = year
450 451 452 453 454 455 456

        # look for an existing record
        rec_id = get_id(db.publications, **kwargs)
        if not rec_id:
            return (None, 0)

        # fix origin field
457 458
        publication = db.publications[rec_id]
        ok = publication.origin and publication.origin == oai_url
459 460
        if not ok:
            if not self.dry_run:
461
                publication = dict(origin=oai_url)
462

463
            logs[-1].modify(MSG_FIX_ORIGIN, year)
464 465
            return (rec_id, 1)

466
        logs[-1].idle(MSG_IN_DB, year)
467 468
        return (rec_id, 0)

469 470
    def insert_record(self, record):
        """Insert the record in the database.
471

472 473 474
        Note:
            This method depend on the type of publications.
            It has to be implemented for each inherited class.
475

476
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
477 478
            record (Record):
                record describing the publication.
479

480
        Returns:
LE GAC Renaud's avatar
LE GAC Renaud committed
481 482
            int:
                one when the record is inserted / updated in the database,
483
                zero otherwise.
484 485 486 487

        """
        return 0

LE GAC Renaud's avatar
LE GAC Renaud committed
488 489 490
    def process_collection(self, collection):
        """"Retrieve JSON objects from the invenio store and for the given
        collection. Corresponding records are inserted in the database.
491

492
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
493 494
            collection (unicode):
                name of the collection to be interrogated.
495

496
        Raises:
LE GAC Renaud's avatar
LE GAC Renaud committed
497 498 499 500 501
            CdsException:
                * keyword argument is invalid;
                * the server return an HTTP error;
                * JSON object can't be decoded
                * not well formed list of ids.
502 503 504

        """
        if self.dbg:
LE GAC Renaud's avatar
LE GAC Renaud committed
505
            print("\nprocess collection", collection)
506 507

        # alias
508
        collection_logs = self.collection_logs
509
        controller = self.controller
LE GAC Renaud's avatar
LE GAC Renaud committed
510
        host = self.harvester.host
511
        project = self.db.projects[self.id_project].project
LE GAC Renaud's avatar
LE GAC Renaud committed
512
        store = self.store
513

514 515
        # log collection information
        # A collection is identified as "Project Controller collection"
LE GAC Renaud's avatar
LE GAC Renaud committed
516 517
        ctitle = "%s / %s / %s" % (project, controller, collection)
        collection_logs.append(MsgCollection(title=ctitle))
518

LE GAC Renaud's avatar
LE GAC Renaud committed
519
        # get search parameters for the collection including user criteria
520
        kwargs = self._search_parameters(collection)
521

LE GAC Renaud's avatar
LE GAC Renaud committed
522
        # get the list of record identifier matching the search criteria
523 524
        try:
            rec_ids = store.get_ids(**kwargs)
525

LE GAC Renaud's avatar
LE GAC Renaud committed
526
        except CdsException as error:
527 528 529
            collection_logs[-1].url = store.last_search_url()
            collection_logs[-1].error = error
            return
530

LE GAC Renaud's avatar
LE GAC Renaud committed
531
        # log the number of record found for the collection
532 533
        collection_logs[-1].url = store.last_search_url()
        collection_logs[-1].found = len(rec_ids)
534

LE GAC Renaud's avatar
LE GAC Renaud committed
535 536 537
        if len(rec_ids) == 0:
            if self.dbg:
                print("\tNo records found in %s" % collection)
538
            return
539

540
        if self.dbg:
LE GAC Renaud's avatar
LE GAC Renaud committed
541
            print("\t%i records found in %s" % (len(rec_ids), collection))
542

LE GAC Renaud's avatar
LE GAC Renaud committed
543 544 545 546
        # remove form the list identifier already registered in the data base
        # and log them
        func = self._is_record_in_db
        rec_ids = [el for el in rec_ids if func(ctitle, host, el) == 0]
547

LE GAC Renaud's avatar
LE GAC Renaud committed
548 549 550 551 552 553 554 555 556 557 558
        # process the remaining identifiers
        [self.process_recid(rec_id) for rec_id in rec_ids]

    def process_recid(self, rec_id):
        """Process the publication:

            * get the publication data from the store using its identifier
            * instantiate the record (RecordPubli, REcordConf, RecordThesis)
            * process OAI data
            * check the record
            * insert new record in the database
559

560
        Args:
LE GAC Renaud's avatar
LE GAC Renaud committed
561 562
            rec_id (int):
                identifier of the publication in the store.
563

LE GAC Renaud's avatar
LE GAC Renaud committed
564 565 566 567
        Raise:
            CdsException:
                * the server return an HTTP error.
                * no JSON object could be decoded.
568

LE GAC Renaud's avatar
LE GAC Renaud committed
569
        """
570
        if self.dbg:
LE GAC Renaud's avatar
LE GAC Renaud committed
571
            print("\nprocessing record", rec_id)
572

LE GAC Renaud's avatar
LE GAC Renaud committed
573 574
        collection_logs = self.collection_logs
        harvester = self.harvester
575 576
        logs = self.logs

LE GAC Renaud's avatar
LE GAC Renaud committed
577 578 579 580
        # instantiate the record
        try:
            recjson = self.store.get_record(rec_id)
            record = build_record(recjson)
581 582

            if self.dbg:
LE GAC Renaud's avatar
LE GAC Renaud committed
583
                print("\t", record.title())
584

LE GAC Renaud's avatar
LE GAC Renaud committed
585 586 587 588 589 590 591 592
        except Exception as e:
            print(traceback.format_exc())
            url = OAI_URL % (harvester.host, rec_id)
            logs.append(Msg(harvester=harvester,
                            collection=collection_logs[-1].title,
                            record_id=rec_id,
                            title=url))
            logs[-1].reject(e)
LE GAC Renaud's avatar
LE GAC Renaud committed
593
            return
LE GAC Renaud's avatar
LE GAC Renaud committed
594 595 596 597 598 599 600 601 602 603

        # start the log for the record
        logs.append(Msg(harvester=harvester,
                        collection=collection_logs[-1].title,
                        record_id=record.id(),
                        title=record.title()))

        # check that the record is well formed
        # repair non-conformity as far as possible
        if not self.check_record(record):
604
            if self.dbg:
LE GAC Renaud's avatar
LE GAC Renaud committed
605 606 607 608 609
                print("\trecord rejected", logs[-1].txt)
                return

        if self.dbg:
            print("\tinsert record in the database")
610

LE GAC Renaud's avatar
LE GAC Renaud committed
611 612
        # insert the record in the database
        self.insert_record(record)
613

LE GAC Renaud's avatar
LE GAC Renaud committed
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630
        if self.dbg:
            log = logs[-1]
            action = log.action
            action = (action.upper() if isinstance(action, str) else action)
            print("\tlog:", action, log.txt)

    def process_url(self, host, collections):
        """Retrieve JSON objects from the invenio store and
        insert corresponding records in the database.

        Args:
            host (unicode):
                host name to query for publications, either
                ``cds.cern.ch`` or ``inspirehep.net``.

            collections (unicode):
                list of collection to be interrogated.
631
                Collections are separated by a comma.
LE GAC Renaud's avatar
LE GAC Renaud committed
632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658

        Raises:
           StoreException:
               when something goes wrong interrogating the store.

           CheckException:
               when the record has non-conformities.

           Exception:
               when the python code crashes.

        """
        if self.dbg:
            print("process URL search")

        # extend harvester for logs
        self.harvester.host = host
        self.harvester.collections = collections

        # instantiate the store
        self.store = InvenioStore(host)

        # list of collections
        collections = re.sub(" *, *", ",", collections).split(",")

        # process
        [self.process_collection(collection) for collection in collections]
659 660 661 662

    def report(self):
        """Build the processing report.

663 664
        Returns:
            dict:
LE GAC Renaud's avatar
LE GAC Renaud committed
665 666
                * ``collection_logs`` list of :class:`MsgCollection`
                * ``controller`` unicode
667
                * ``logs`` list of :class:Msg
LE GAC Renaud's avatar
LE GAC Renaud committed
668
                * ``selector`` :class:`plugin_dbui.Selector`
669 670 671 672 673 674

        """

        return dict(collection_logs=self.collection_logs,
                    controller=self.controller,
                    logs=self.logs)