Commit 2d638360 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production'

Release 0.9.7.1

See merge request !96
parents a50cc714 92ae4e31
......@@ -2,11 +2,19 @@
HEAD
0.9.7.1 (Oct 2019)
- Minor release.
- Require plugin_dbui 0.9.9.0 or higher.
- More strict rule to select iauthors per affiliation.
- Add a Go button in filter associated to grid.
- Restore the user role.
- Unlock the table authors_roles.
0.9.7.0 (Oct 2017)
- Require plugin_dbui 0.9.8.3 or higher.
- Increase the length of the field harvesters.collections to 1024
- Increase the length of the field harvesters.collections to 1024
instead of 255.
- Change the rule for the field harvesters.collections. It now contains
- Change the rule for the field harvesters.collections. It now contains
the definiton for a single collection (it was possible to define several
collection separated by a comma but this confict with author encoding).
- Fix several bugs in the documentation and in the wizard harvesters.
......
0.9.7.0
\ No newline at end of file
0.9.7.1
\ No newline at end of file
......@@ -65,7 +65,7 @@ def index():
li = [el for el in section.orderby]
field = db.publications[section.sort_field]
if section.direction == "DESC":
if section.direction == "DESC":
field = ~field
li.append(field)
......
......@@ -573,7 +573,7 @@
'Rejected': 'Rejeté',
"Rejeté la chaine XML n'est pas correcte": "Rejeté la chaine XML n'est pas correcte",
'relation harvesters / categories': 'relation moissoneurs / catégories',
'Relation team / projects': 'Relation équipe / projets',
'relation team / projects': 'relation équipe / projets',
'relation user / groups': 'relation utilisateur / groupes',
'Removing affiliation failed. Rules might be missing for the given string. Contact the author.': 'Removing affiliation failed. Rules might be missing for the given string. Contact the author.',
'Render': 'Moteur de rendu',
......
......@@ -16,7 +16,7 @@ DEF_ADMIN = "administrators, librairians,..."
# constant for user role
ID_USER = 2
USER = "user",
USER = "user"
DEF_USER = "liaisons, team leaders,..."
......
......@@ -351,6 +351,7 @@ def query_publications(db, is_publish, **kwargs):
query = (qi if query is None else (query) & (qi))
query = (publications.id > 0 if query is None else query)
return query
......
......@@ -569,7 +569,7 @@ class Automaton(object):
if db_id:
if self.dbg:
print "record in db", rec_id, "->", db_id
return
continue
xml = store.get_record(rec_id)
decode_xml(xml)
......@@ -611,7 +611,7 @@ class Automaton(object):
if self.dbg:
print "record decoded"
print record.title()
print record.title().encode("utf-8")
# reject the record using the secondary OAI
# require to cover the case:
......
......@@ -110,23 +110,28 @@ class CheckAndFix(object):
app = current.app
reg_institute = app.reg_institute
# regular expression for the affiliation is not defined
# build it using the affiliation keys.
# regular expression for the affiliation keys
# protect special character
# add start and end of string for an exact match
if not reg_institute:
iterselect = db(db.affiliation_keys.id > 0).iterselect()
reg_institute = \
u"|".join(imap(
lambda row: u"%(key_u)s%(key_v)s" % row, iterselect))
lst = []
for row in db(db.affiliation_keys.id > 0).iterselect():
val = row.key_u
# protect special character
reg_institute = (reg_institute
.replace("(", "\\(")
.replace(")", "\\)")
.replace("&", "\\&")
.replace("$", "\\$")
.replace("+", "\\+")
.replace("?", "\\?"))
val = (val
.replace("(", "\(")
.replace(")", "\)")
.replace("&", "\&")
.replace("$", "\$")
.replace("+", "\+")
.replace("?", "\?"))
val = r"(^|\|){}($|\|)" .format(val)
lst.append(val)
reg_institute = r"|".join(lst)
return reg_institute
......
......@@ -312,21 +312,21 @@ class RecordPubli(Record):
Args:
pattern (unicode):
regular expression defining the affiliation keys.
It should not contains groups.
It has to be build for an exact match namely containing
start and end of string. This is reuqired to separate
`Ecole Plolytechnique` from `Ecole Polytechnique, Lausanne`.
Returns:
unicode:
- the affiliation
- the affiliation or the first one when several are found.
- empty string when nothing is found.
"""
# modify the pattern to capture group
pattern = "(%s)" % pattern
data = (self[u"700"].u.str.extract(pattern, expand=False)
.dropna())
df = self[u"700"]
query = df.u.str.contains(pattern)
data = (df[query].u.unique())
return (data.iloc[0] if len(data) > 0 else u"")
return (data[0] if len(data) > 0 else u"")
def find_authors(self, pattern, sep=u", ", sort=False):
"""Find authors containing the regular expression *pattern*.
......@@ -385,7 +385,6 @@ class RecordPubli(Record):
"""
df = self[u"700"]
query = df.u.str.contains(pattern)
if sort:
......
......@@ -73,7 +73,7 @@ class CoreUI(object):
#
# Grid
#
mdf = GridModifier("authors_role")
mdf = GridModifier("authors_roles")
mdf.configure_column("role", width=75)
# ....................................................................
......
......@@ -225,9 +225,9 @@ class ViewportUi(object):
node.add_children(leaves, func=to_grid)
node.add_child(
T("Relation team / projects"),
to_grid("organisation"))
add_child = node.add_child
add_child(T("relation team / projects"), to_grid("organisation"))
add_child(T("authors_roles"), to_grid("authors_roles"))
return node
......
......@@ -514,7 +514,7 @@ CREATE TABLE `preferences` (
LOCK TABLES `preferences` WRITE;
/*!40000 ALTER TABLE `preferences` DISABLE KEYS */;
INSERT INTO `preferences` VALUES (1,'authorize_harvester_scan','false','Les moissonneurs sont éxécuté automatiquement quand cette valuer est égale à vraie.'),(2,'authorize_user_login','true','Les utilisateurs peuvent se logger quand cette valuer est égale à vraie.'),(3,'harvester_start_year','2010','Défini les années pour lesquelles les moissonneurs peuvent être éxecutés. Le début de la période est la valeur donnée. La fin de la période est l\'année courrante.');
INSERT INTO `preferences` VALUES (1,'authorize_harvester_scan','false','Les moissonneurs sont éxécuté automatiquement quand cette valuer est égale à vraie.'),(2,'authorize_user_login','true','Les utilisateurs peuvent se logger quand cette valuer est égale à vraie.'),(3,'harvester_start_year','2010','Défini les années pour lesquelles les moissonneurs peuvent être éxecutés. Le début de la période est la valeur donnée. La fin de la période est l\'année courrante.'),(4,'dashboard_start_year','2016','Année de début du tableau de bord');
/*!40000 ALTER TABLE `preferences` ENABLE KEYS */;
UNLOCK TABLES;
......
......@@ -34,6 +34,7 @@ fermi-lat,articles,inspirehep.net,find cn fermi-lat and tc p and not tc c,
fopi,articles,inspirehep.net,find cn fopi and tc p and not tc c,
gaspard,articles,inspirehep.net,find cn gaspard and tc p and not tc c,
graal,articles,inspirehep.net,find cn graal and tc p and not tc c,
geant4-dna,articles,inspirehep.net,find cn geant4-dna and tc p and not tc c,
hades,articles,inspirehep.net,find cn hades and tc p and not tc c,
h1,articles,inspirehep.net,find cn h1 and tc p and not tc c,
hess,articles,inspirehep.net,find (cn hess or cn h e s s) and tc p and not tc c,
......@@ -99,6 +100,7 @@ fcc,proceedings,inspirehep.net,find fcc and tc c,
fermi-lat,proceedings,inspirehep.net,find cn fermi-lat and tc c,
fopi,proceedings,inspirehep.net,find cn fopi and tc c,
gaspard,proceedings,inspirehep.net,find cn gaspard and tc c,
geant4-dna,proceedings,inspirehep.net,find cn geant4-dna and tc c,
graal,proceedings,inspirehep.net,find cn graal and tc c,
hades,proceedings,inspirehep.net,find cn hades and tc c,
h1,proceedings,inspirehep.net,find cn h1 and tc c,
......
......@@ -43,7 +43,14 @@
EXAMPLE
> cd ...limbra/scripts
> run -S test_limbra script deploy.py data/my_institute.csv
Dry run:
> run -S test_limbra script deploy.py -t data/my_institute.csv
> run -S test_limbra script deploy.py -a data/my_institute.csv
> run -S test_limbra script deploy.py -H data/my_institute.csv
Commit change in the database:
> run -S test_limbra script deploy.py -taHc data/my_institute.csv
AUTHOR
R. Le Gac -- Jul 2017
......@@ -76,19 +83,15 @@ def create_affiliation(opt):
opt (argparse.Namespace): options
"""
rep = raw_input("\tDo you want to create affiliation keys [y/N]: ")
if rep != "y":
return
rep = raw_input("\tEnter the inspirehep identifier for your institute: ")
if not rep.isdigit():
aff = raw_input("\tEnter the inspirehep identifier for your institute: ")
if not aff.isdigit():
print "\t\tERROR: invalid answer!"
# ........................................................................
#
# get and process inspirehep record
#
record = load_record("inspirehep.net", int(rep))
record = load_record("inspirehep.net", aff)
# extract keys defining the affiliation
# u and v are the main keys use in inspirehep and cds
......@@ -108,14 +111,6 @@ def create_affiliation(opt):
if not is_key_add:
print "\t\tAffiliation keys already exist!"
# ........................................................................
#
# commit
#
rep = raw_input("\n\tDo you want to commit database changes [y/N]: ")
if rep == "y":
db.commit()
def create_harvesters(opt, dfi, dfh):
"""Create the harvesters in the database.
......@@ -142,10 +137,6 @@ def create_harvesters(opt, dfi, dfh):
- collection
"""
rep = raw_input("\tDo you want to create harvesters [y/N]: ")
if rep != "y":
return
# ........................................................................
#
# check that all collaborations are defined
......@@ -271,37 +262,37 @@ def create_harvesters(opt, dfi, dfh):
data["controller"] = "articles"
li = ["a %s" % el.strip() for el in people.split(",")]
collection = "find (%s) and tc p and not tc c " % " or ".join(li)
if len(row.hal_hidden) > 0:
collection = collection.replace("find", "find cc Hal Hidden")
auths = " or ".join(li)
collection = "find %s and tc p and not tc c " % auths
data["collections"] = collection
data["id_categories"] = CONTROLLER_TO_CAT["articles"]
insert_harvester(**data)
# systematic search in hall hidden
collection = "find cc Hal Hidden and %s and not tc c " % auths
data["collections"] = collection
insert_harvester(**data)
# proceeding
data["controller"] = "proceedings"
collection = "find (%s) and tc c " % " or ".join(li)
collection = "find %s and tc c " % auths
data["collections"] = collection
if len(row.hal_hidden) > 0:
collection = collection.replace("find", "find cc Hal Hidden")
data["id_categories"] = CONTROLLER_TO_CAT["proceedings"]
insert_harvester(**data)
# systematic search in hall hidden
collection = "find cc Hal Hidden and %s and tc c " % auths
data["collections"] = collection
data["id_categories"] = CONTROLLER_TO_CAT["proceedings"]
insert_harvester(**data)
continue
# ........................................................................
#
# commit
#
rep = raw_input("\n\tDo you want to commit database changes [y/N]: ")
if rep == "y":
db.commit()
def create_teams_projects(opt, dfi):
"""Create the teams / projects structure in the database.
......@@ -311,10 +302,6 @@ def create_teams_projects(opt, dfi):
dfi (pandas.DataFrame): institute parameter
"""
rep = raw_input("\tDo you want to create teams/projects [y/N]: ")
if rep != "y":
return
# ........................................................................
#
# teams / projects
......@@ -359,14 +346,6 @@ def create_teams_projects(opt, dfi):
else:
print "\t\tRelation %s/%s already exist!" % (team, project)
# ........................................................................
#
# commit
#
rep = raw_input("\n\tDo you want to commit database changes [y/N]: ")
if rep == "y":
db.commit()
def harvesters(opt):
"""Create the DataFrame with the harvesters definitions.
......@@ -392,6 +371,10 @@ def harvesters(opt):
"scripts",
opt.path_harvesters)
if not os.path.exists(path):
print "\n\tFile {0} don't exist!".format(path)
sys.exit(1)
dh = (pd.read_csv(path)
.dropna(how="all", axis="index")
.drop("comment", axis="columns"))
......@@ -455,6 +438,10 @@ def institute(opt):
"scripts",
opt.institute)
if not os.path.exists(path):
print "\n\tFile {0} don't exist!".format(path)
sys.exit(1)
df = pd.read_csv(path)
# rename columns
......@@ -480,8 +467,6 @@ def institute(opt):
if __name__ == "__main__":
print "\n%s\nStart deploy" % ("."*80,)
# pandas options and matplotlib style
pd.set_option("display.encoding", "utf-8")
pd.set_option("display.width", 150)
......@@ -495,10 +480,9 @@ if __name__ == "__main__":
help="create the affiliation keys [%(default)s].")
PARSER.add_argument(
"--all",
"-c", "--commit",
action="store_true",
help="create everything [%(default)s]. "
"Equivalent to --affiliation --teams --harvesters.")
help="commit change to the database [%(default)s]. ")
PARSER.add_argument(
"-H", "--harvesters",
......@@ -526,17 +510,31 @@ if __name__ == "__main__":
OPT = PARSER.parse_args()
DFH = harvesters(OPT)
if not (OPT.harvesters or OPT.teams or OPT.affiliation):
print "\n\tPlease select an option!"
print "\tPossible ones are --affiliation, --harvesters or --teams"
sys.exit(1)
print "\n%s\nStart deploy" % ("."*80,)
DFI = institute(OPT)
DFH = harvesters(OPT)
if OPT.teams or OPT.all:
if OPT.teams:
create_teams_projects(OPT, DFI)
if OPT.harvesters or OPT.all:
if OPT.harvesters:
create_harvesters(OPT, DFI, DFH)
if OPT.affiliation or OPT.all:
if OPT.affiliation:
create_affiliation(OPT)
if OPT.commit:
print "\n\tChange are committed to the database."
db.commit()
else:
print "\n\tDry run, the database is not modified."
print "\nEnd deploy\n%s\n" % ("."*80, )
sys.exit(0)
......@@ -39,19 +39,21 @@ if __name__ == "__main__":
collections = row.collections
if row.host == "cds.cern.ch" and "," in collections:
print "\n\tsplit:", collections
data = row.as_dict()
del data["id"]
# insert new harvesters
for elt in collections.split(","):
print "\t\tinsert →", elt.strip()
data["collections"] = elt.strip()
db.harvesters.insert(**data)
# delete old harvesters
print "\t\tdelete →", row.id
del db.harvesters[row.id]
rep = raw_input("\n\tSplit '%s'? [y/N]: " % collections)
if rep == "y":
print "\n\tsplit:", collections
data = row.as_dict()
del data["id"]
# insert new harvesters
for elt in collections.split(","):
print "\t\tinsert →", elt.strip()
data["collections"] = elt.strip()
db.harvesters.insert(**data)
# delete old harvesters
print "\t\tdelete →", row.id
del db.harvesters[row.id]
# commit change
rep = raw_input("Commit database changes [y/N]: ")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment