Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 78090b4d authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Migrate controller Insert_MARCXML to Migrate_RECJSON.

parent d9027694
""" Harvest Controllers
"""
import json
import traceback
from gluon import current
......@@ -10,10 +10,10 @@ from harvest_tools import (build_harvester_tool,
CheckAndFix,
CheckException,
DRY_RUN,
MsgCollection,
search_synonym,
ToolException)
from invenio_tools import (CdsException,
load_record,
from invenio_tools import (load_record,
OAI_URL,
RecordConf,
RecordThesis)
......@@ -292,15 +292,15 @@ def edit_insert():
return dict(cfg=cfg, values=values)
def insert_marcxml():
"""Insert a MarcXML record in the database.
def insert_recjson():
"""Insert a recjson record in the database.
"""
if db(db.affiliation_keys.id > 0).count() == 0:
return inline_alert(T("Error"), T(MSG_NO_AFFILIATION))
try:
selector = Selector(virtdb.marc12_selector, exclude_fields=("mode"))
selector = Selector(virtdb.recjson_selector, exclude_fields=("mode"))
tool = build_harvester_tool(
db,
......@@ -316,8 +316,17 @@ def insert_marcxml():
if not tool:
return inline_alert(T("Error"), T("Select an harvester."))
ctitle = "%s / %s " % (db.projects[selector.id_projects].project,
selector.controller)
tool.collection_logs = [MsgCollection(title=ctitle)]
tool.harvester.host = selector.host
tool.process_xml(selector.xml)
tool.logs = []
recjson = json.loads(selector.recjson)
recjson = (recjson[0] if isinstance(recjson, list) else recjson)
tool.process_recjson(recjson)
except ToolException as e:
return T(str(e))
......
......@@ -45,7 +45,7 @@ class Selector(object):
Selector.graph_selector(virtdb, db, T)
Selector.harvester_selector(virtdb, db, T)
Selector.list_selector(virtdb, db, T)
Selector.marc12_selector(virtdb, db, T)
Selector.recjson_selector(virtdb, db, T)
Selector.metric_selector(virtdb, db, T)
Selector.run_all_harvesters_selector(virtdb, db, T)
......@@ -338,8 +338,8 @@ class Selector(object):
return table
@staticmethod
def marc12_selector(virtdb, db, T):
"""marc12 selector.
def metric_selector(virtdb, db, T):
"""metric selector.
Note:
selector table are store in a virtual database.
......@@ -353,34 +353,30 @@ class Selector(object):
pyDAL.Table
"""
mode_dry_run = T(DRY_RUN)
tp_xml = T("The XML string describing the MARC12 record. "
'It Starts with "<?xml" and ends with "</collection>"')
year = current.request.now.year
table = virtdb.define_table(
"marc12_selector",
"metric_selector",
Field("year_start", "integer", default=year),
Field("year_end", "integer"),
Field("id_teams", "reference teams", label="Team"),
Field("id_projects", "reference projects", label="Project"),
Field("host", "string", default=STORES[0], label="Store"),
Field("xml", "text", comment=tp_xml, label="MARCXML"),
Field("controller", "string", label="Automaton"),
Field("id_categories", "reference categories", label="Category"),
Field("mode", "string", default=mode_dry_run))
Field("author", "string"),
Field("id_authors_roles", "reference authors_roles", label="Role"),
Field("id_metrics", "reference metrics", label="Metric"))
table.controller.requires = IS_IN_SET(CONTROLLERS)
table.host.requires = IS_IN_SET(STORES)
table.id_authors_roles.requires = IS_IN_DB(db, "authors_roles.role")
table.id_categories.requires = IS_IN_DB(db, "categories.code")
table.id_metrics.requires = IS_IN_DB(db, "metrics.metric")
table.id_projects.requires = IS_IN_DB(db, "projects.project")
table.id_teams.requires = IS_IN_DB(db, "teams.team")
table.mode.requires = IS_IN_SET([mode_dry_run, T(MODE_LOAD_IN_DB)])
table.xml.requires = IS_MATCH("<\?xml*")
return table
@staticmethod
def metric_selector(virtdb, db, T):
"""metric selector.
def recjson_selector(virtdb, db, T):
"""recjson selector.
Note:
selector table are store in a virtual database.
......@@ -394,24 +390,26 @@ class Selector(object):
pyDAL.Table
"""
year = current.request.now.year
mode_dry_run = T(DRY_RUN)
tp_xml = T("The JSON string describing the record.")
table = virtdb.define_table(
"metric_selector",
Field("year_start", "integer", default=year),
Field("year_end", "integer"),
"recjson_selector",
Field("id_teams", "reference teams", label="Team"),
Field("id_projects", "reference projects", label="Project"),
Field("host", "string", default=STORES[0], label="Store"),
Field("recjson", "text", comment=tp_xml, label="RECJSON"),
Field("controller", "string", label="Automaton"),
Field("id_categories", "reference categories", label="Category"),
Field("author", "string"),
Field("id_authors_roles", "reference authors_roles", label="Role"),
Field("id_metrics", "reference metrics", label="Metric"))
Field("mode", "string", default=mode_dry_run))
table.id_authors_roles.requires = IS_IN_DB(db, "authors_roles.role")
table.controller.requires = IS_IN_SET(CONTROLLERS)
table.host.requires = IS_IN_SET(STORES)
table.id_categories.requires = IS_IN_DB(db, "categories.code")
table.id_metrics.requires = IS_IN_DB(db, "metrics.metric")
table.id_projects.requires = IS_IN_DB(db, "projects.project")
table.id_teams.requires = IS_IN_DB(db, "teams.team")
table.mode.requires = IS_IN_SET([mode_dry_run, T(MODE_LOAD_IN_DB)])
return table
......
......@@ -27,7 +27,7 @@ class SelectorUI(object):
SelectorUI.graph_selector(virtdb, db, T)
SelectorUI.harvester_selector(virtdb, db, T)
SelectorUI.list_selector(virtdb, db, T)
SelectorUI.marc12_selector(virtdb, db, T)
SelectorUI.recjson_selector(virtdb, db, T)
SelectorUI.metric_selector(virtdb, db, T)
SelectorUI.run_all_harvesters_selector(virtdb, db, T)
......@@ -259,36 +259,6 @@ class SelectorUI(object):
emptyText=tr_select,
xtype="xcomboboxuserreset")
@staticmethod
def marc12_selector(virtdb, db, T):
"""UI for the marc12_selector table
Args:
virtdb (pyDAL.DAL): connection to the virtual database
db (pyDAL.DAL): database connection
T (gluon.languages.translator): language translator
"""
# ....................................................................
#
# Form
#
marc12_selector = virtdb.marc12_selector
# Relation between teams and projects
mdf = FormModifier("marc12_selector")
mdf.link_comboboxes(master=marc12_selector.id_projects,
slave=marc12_selector.id_teams,
masterHasSlaveData="organisation",
slaveExtJS={"hidden": True})
# Relation between harvester controller and publication categories
mdf.link_comboboxes(master=marc12_selector.controller,
slave=marc12_selector.id_categories,
masterHasSlaveData="controllers",
slaveExtJS={"hidden": False})
@staticmethod
def metric_selector(virtdb, db, T):
"""UI for the metric_selector table
......@@ -330,6 +300,36 @@ class SelectorUI(object):
emptyText=tr_select,
xtype="xcomboboxuserreset")
@staticmethod
def recjson_selector(virtdb, db, T):
"""UI for the recjson_selector table
Args:
virtdb (pyDAL.DAL): connection to the virtual database
db (pyDAL.DAL): database connection
T (gluon.languages.translator): language translator
"""
# ....................................................................
#
# Form
#
recjson_selector = virtdb.recjson_selector
# Relation between teams and projects
mdf = FormModifier("recjson_selector")
mdf.link_comboboxes(master=recjson_selector.id_projects,
slave=recjson_selector.id_teams,
masterHasSlaveData="organisation",
slaveExtJS={"hidden": True})
# Relation between harvester controller and publication categories
mdf.link_comboboxes(master=recjson_selector.controller,
slave=recjson_selector.id_categories,
masterHasSlaveData="controllers",
slaveExtJS={"hidden": False})
@staticmethod
def run_all_harvesters_selector(virtdb, db, T):
"""UI for the run_all_harvesters_selector table
......
......@@ -473,10 +473,10 @@ class ViewportUi(object):
baseUrl=URL("harvest", "edit_insert"),
selectorTitle="Fill all fields")
marc12_panel = to_panelWithUrlSelector(
virtdb.marc12_selector,
recjson_panel = to_panelWithUrlSelector(
virtdb.recjson_selector,
selectorTitle="Fill all fields",
baseUrl=URL("harvest", "insert_marcxml"))
baseUrl=URL("harvest", "insert_recjson"))
add_child = node.add_child
add_child(T("extract authors", lazy=False), author_panel)
......@@ -487,7 +487,7 @@ class ViewportUi(object):
add_child(T("check", lazy=False), check_panel)
add_child(T("edit and insert", lazy=False), edit_panel)
add_child(T("insert MARCXML", lazy=False), marc12_panel)
add_child(T("insert RECJSON", lazy=False), recjson_panel)
node.sort_children()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment