Commit 304afdca authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Implement Selector in harvester tools and redesign the harvester

controllers.
parent a27595a8
......@@ -12,80 +12,40 @@ from harvest_tools import (Articles,
ToolException,
Thesis)
def index():
return T('Select an "harvester" !!!')
from plugin_dbui import Selector
def articles():
"""Scan the cds/invenio stores to find articles published during year range
and for a given team. Insert them in the database if they don't exist.
def insert_marcxml():
"""Insert a MarcXML record in the database.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
Search arguments are defined and via the harvester selector.
"""
try:
tool = Articles(debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return tool.report()
def notes():
"""Scan the cds/invenio stores to find notes published during a year
range and for a given team. Insert them in the database if they don't
exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
selector = Selector(virtdb.marc12_selector)
Search arguments are defined and via the harvester selector.
if selector.controller == 'articles':
Tool = Articles
"""
try:
tool = Notes(debug=False)
tool.process(db)
elif selector.controller == 'notes':
Tool = Notes
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return tool.report()
def preprints():
"""Scan the cds/invenio stores to find preprints not yet published.
the scan is performed during a year range for a given team.
New record are inserted in the database if they don't exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
Search arguments are defined and via the harvester selector.
"""
try:
tool = Preprints(debug=False)
elif selector.controller == 'preprints':
Tool = Preprints
elif selector.controller == 'proceedings':
Tool = Proceedings
elif selector.controller == 'reports':
Tool = Reports
elif selector.controller == 'talks':
Tool = Talks
elif selector.controller == 'theses':
Tool = Thesis
tool = Tool(selector, debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
......@@ -99,76 +59,45 @@ def preprints():
return tool.report()
def proceedings():
"""Scan the cds/invenio stores to find conference proceedings published
during a year range for a given team. Insert them in the database
if they don't exist.
def run():
"""Scan the cds/invenio stores to find articles published during
a given range of years and for a given team/project.
Insert them in the database if they don't exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
Search arguments are defined and via the harvester selector.
"""
try:
tool = Proceedings(debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return tool.report()
def reports():
"""Scan the cds/invenio stores to find reports published during a year
range and for a given team. Insert them in the database if they don't
exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
Search arguments are defined via the harvester selector.
Search arguments are defined and via the harvester selector.
"""
try:
tool = Reports(debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
selector = Selector(virtdb.harvester_selector,
exclude_fields=('year_start', 'year_end'))
response.view = 'harvest/layout.html'
return tool.report()
def talks():
"""Scan the cds/invenio stores to find conference talks for a given
year range and team. Insert them in the database if they don't exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
if selector.controller == 'articles':
Tool = Articles
Search arguments are defined and via the harvester selector.
"""
try:
tool = Talks(debug=False)
tool.process(db)
elif selector.controller == 'notes':
Tool = Notes
elif selector.controller == 'preprints':
Tool = Preprints
elif selector.controller == 'proceedings':
Tool = Proceedings
elif selector.controller == 'reports':
Tool = Reports
elif selector.controller == 'talks':
Tool = Talks
elif selector.controller == 'theses':
Tool = Thesis
tool = Tool(selector, debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
......@@ -177,34 +106,8 @@ def talks():
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return tool.report()
def theses():
"""Scan the cds/invenio stores to find theses.
the scan is performed during a year range for a given team.
New record are inserted in the database if they don't exist.
The scanning is steered using the current request arguments as well as
the harvest parameters associated to this action.
Search arguments are defined and via the harvester selector.
"""
try:
tool = Thesis(debug=False)
tool.process(db)
except ToolException, e:
return T(str(e))
except BaseException, e:
msg = '<br><br><hr/>'
msg += CODE(traceback.format_exc()).xml()
msg += '<hr/>'
return msg
response.view = 'harvest/layout.html'
return tool.report()
......@@ -91,14 +91,12 @@ authorNode = dbui.to_panelWithUrlSelector(virtdb.authors_selector,
checkNode = dbui.to_panelWithUrlSelector(virtdb.check_selector,
baseUrl=URL('wizards', 'check_validate'))
url = '/'+request.application+'/harvest'
harvesterNode = dbui.to_panelWithUrlSelector(virtdb.harvester_selector,
baseUrl=url,
ctrlField='Harvester_selectorController',
baseUrl=URL('harvest', 'run'),
timeout=3600)
url = '/'+request.application+'/harvest'
marc12Node = dbui.to_panelWithUrlSelector(virtdb.marc12_selector, baseUrl=url)
marc12Node = dbui.to_panelWithUrlSelector(virtdb.marc12_selector,
baseUrl=URL('harvest', 'insert_marcxml'))
wizardNode = dbui.Node(T('Wizards'))
wizardNode.add_child(T('insert MARCXML'), marc12Node)
......
This diff is collapsed.
<p class="my-p my-small-cap my-large">{{=controller.title()}} search:</p>
<p class="my-p my-small-cap my-large">{{=harvester.controller.title()}} search:</p>
<br>
<ul class="my-ul-none">
<li class="my-li">&ndash; Look for {{=controller}} in {{=', '.join(collections)}} </li>
{{if y1 and not y2:}}
<li class="my-li">&ndash; Scan {{=y1}}</li>
{{elif y1 and y2:}}
<li class="my-li">&ndash; Scan between {{=y1}} and {{=y2}}<li>
<li class="my-li">&ndash; Look for {{=harvester.controller}} in {{=harvester.collections}} </li>
{{if selector.year_start and not selector.year_end:}}
<li class="my-li">&ndash; Scan {{=selector.year_start}}</li>
{{elif selector.year_start and selector.year_end:}}
<li class="my-li">&ndash; Scan between {{=selector.year_start}} and {{=selector.year_end}}<li>
{{pass}}
<br>
<li class="my-li">&ndash; Number of records found: {{=nfound}}</li>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment