Commit 59f403fe authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

The new controller run_all_harvesters in running.

parent 40c1f5fe
......@@ -3,6 +3,7 @@
"""
import traceback
from gluon.storage import Storage
from harvest_tools import get_harvester_tool, ToolException
from plugin_dbui import Selector
......@@ -75,4 +76,48 @@ def run_all():
"""Run all harvesters in one go.
"""
return 'hello'
collections = []
logs = []
nfound = 0
ninsert = 0
urls = []
try:
selector = Selector(virtdb.run_all_harvesters_selector,
exclude_fields=('year_start', 'year_end'))
for row in db().select(db.harvesters.ALL):
selector.controller = row.controller
selector.id_projects = row.id_projects
selector.id_teams = row.id_teams
collections.extend(row.collections.split(','))
Tool = get_harvester_tool(selector.controller)
tool = Tool(selector, debug=False)
tool.process(db)
logs.extend(tool.logs)
nfound += tool.nfound
ninsert += tool.ninsert
urls.extend(tool.search_urls)
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return dict(harvester=Storage(controller='all harvesters',
collections=','.join(collections)),
logs=logs,
nfound=nfound,
ninsert=ninsert,
selector=selector,
urls=urls)
......@@ -959,7 +959,7 @@ class Talks(PublicationsTool):
"""
# get the conference record
cds = invenio_tools.CdsSvc(host=self.host)
cds = invenio_tools.CdsSvc(host=self.harvester.host)
conference = get_conference_record(cds, self.marc12, record)
# conference is either a Record or and error msg
......@@ -997,7 +997,7 @@ class Talks(PublicationsTool):
for row in db(query).select(db.publications.title):
s = difflib.SequenceMatcher(None, title, row.title)
if s.ratio() > self.ratio:
if s.ratio() > self.harvester.ratio:
match = True
break
......
......@@ -3,7 +3,7 @@
HEAD
- Migrate to plugin_dbui 0.4.9.6.
- New tree structure in the viewport.
- Add a wizard check and validate.
- Add a wizard to check and validate and to run all harvesters.
- Deploy the new class Selector in lists and metrics.
- Add fields in the publication filter.
- Add tables axes, controllers and organisation.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment