Commit a5829fba authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Update deploy.py to fix some bug and improve harvesters.csv.

parent 0d3f1bb3
......@@ -14,7 +14,7 @@
"boss","articles","inspirehep.net","find cn boss and tc p and not tc c",
"calice","articles","inspirehep.net","find cn calice and tc p and not tc c",
"celeste","articles","inspirehep.net","find cn celeste and tc p and not tc c",
"ckmfitter","articles","inspirehep.net","find cn ckmfitter and tc p and not tc c",
"ckmfitter","articles","inspirehep.net","find ckmfitter and tc p and not tc c",
"clas","articles","inspirehep.net","find cn clas and tc p and not tc c",
"cms","articles","cds.cern.ch","CMS Papers",
"codalema","articles","inspirehep.net","find cn codalema and tc p and not tc c",
......@@ -29,6 +29,7 @@
"elisa","articles","inspirehep.net","finc 693__e:elisa and tc p and not tc c","an instrument not a coll."
"euclid","articles","inspirehep.net","find cn euclid and tc p and not tc c",
"fazia","articles","inspirehep.net","find cn fazia and tc p and not tc c",
"fcc","articles","inspirehep.net","find fcc and tc p and not tc c",
"fermi-lat","articles","inspirehep.net","find cn fermi-lat and tc p and not tc c",
"gaspard","articles","inspirehep.net","find cn gaspard and tc p and not tc c",
"graal","articles","inspirehep.net","find cn graal and tc p and not tc c",
......@@ -75,7 +76,7 @@
"boss","proceedings","inspirehep.net","find cn boss and tc c",
"calice","proceedings","inspirehep.net","find cn calice and tc c",
"celeste","proceedings","inspirehep.net","find cn celeste and tc c",
"ckmfitter","proceedings","inspirehep.net","find cn ckmfitter and tc c",
"ckmfitter","proceedings","inspirehep.net","find ckmfitter and tc c",
"clas","proceedings","inspirehep.net","find cn clas and tc c",
"cms","proceedings","inspirehep.net","find cn cms and tc c",
"codalema","proceedings","inspirehep.net","find cn codalema and tc c",
......@@ -90,6 +91,7 @@
"elisa","proceedings","inspirehep.net","finc 693__e:elisa and tc c","an instrument not a coll."
"euclid","proceedings","inspirehep.net","find cn euclid and tc c",
"fazia","proceedings","inspirehep.net","find cn fazia and tc c",
"fcc","proceedings","inspirehep.net","find fcc and tc c",
"fermi-lat","proceedings","inspirehep.net","find cn fermi-lat and tc c",
"gaspard","proceedings","inspirehep.net","find cn gaspard and tc c",
"graal","proceedings","inspirehep.net","find cn graal and tc c",
......
......@@ -198,7 +198,7 @@ def create_harvesters(opt, dfi, dfh):
# harvester defined via collaboration
#
collaboration = row.collaboration
if pd.notnull(collaboration):
if len(collaboration) > 0:
query = dfh.collaboration == collaboration
for el in dfh[query].itertuples(index=False):
......@@ -215,12 +215,13 @@ def create_harvesters(opt, dfi, dfh):
# harvester defined via experiment / instrument
#
instrument = row.instrument
if pd.notnull(instrument):
if len(instrument) > 0:
data["controller"] = "articles"
data["host"] = "inspirehep.net"
# article
data["controller"] = "articles"
collection = "693__e:%s and tc p and not tc c" % instrument
data["collections"] = collection
......@@ -228,6 +229,8 @@ def create_harvesters(opt, dfi, dfh):
insert_harvester(**data)
# proceeding
data["controller"] = "proceedings"
collection = "693__e:%s and tc c" % instrument
data["collections"] = collection
......@@ -240,16 +243,17 @@ def create_harvesters(opt, dfi, dfh):
# harvester defined via people
#
people = row.people
if pd.notnull(people):
if len(people) > 0:
data["controller"] = "articles"
data["host"] = "inspirehep.net"
# article
data["controller"] = "articles"
li = ["a %s" % el.strip() for el in people.split(",")]
collection = "find (%s) and tc p and not tc c " % " ".join(li)
collection = "find (%s) and tc p and not tc c " % " or ".join(li)
if pd.notnull(row.hal_hidden):
if len(row.hal_hidden) > 0:
collection = collection.replace("find", "find cc Hal Hidden")
data["collections"] = collection
......@@ -257,9 +261,11 @@ def create_harvesters(opt, dfi, dfh):
insert_harvester(**data)
# proceeding
collection = "find (%s) and tc c " % " ".join(li)
data["controller"] = "proceedings"
collection = "find (%s) and tc c " % " or ".join(li)
if pd.notnull(row.hal_hidden):
if len(row.hal_hidden) > 0:
collection = collection.replace("find", "find cc Hal Hidden")
data["collections"] = collection
......@@ -327,7 +333,7 @@ def create_teams_projects(opt, dfi):
get_id(db.organisation, id_teams=id_team, id_projects=id_project)
if rec_id is None:
print "Create the relation %s/%s" % (team, project)
print "\t\tCreate the relation %s/%s" % (team, project)
db.organisation.insert(id_teams=id_team, id_projects=id_project)
else:
......@@ -440,6 +446,15 @@ def institute(opt):
"people",
"hal_hidden"]
# drop empty row
df = (df.dropna(how="all")
.fillna(""))
# convert to lower case
df["collaboration"] = df.collaboration.str.lower()
df["instrument"] = df.instrument.str.lower()
df["people"] = df.people.str.lower()
return df
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment