Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
limbra
limbra
Commits
d540196f
Commit
d540196f
authored
Nov 14, 2019
by
LE GAC Renaud
Browse files
Apply 2to3 converter to scripts.
parent
eb83aeb7
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
56 additions
and
70 deletions
+56
-70
scripts/_run_pytest.py
scripts/_run_pytest.py
+0
-1
scripts/archives_fix_py27/change_status_submitted.py
scripts/archives_fix_py27/change_status_submitted.py
+0
-0
scripts/deploy.py
scripts/deploy.py
+32
-34
scripts/export_to_csv.py
scripts/export_to_csv.py
+1
-3
scripts/import_from_csv.py
scripts/import_from_csv.py
+1
-3
scripts/limbra_status.py
scripts/limbra_status.py
+2
-3
scripts/statistics.py
scripts/statistics.py
+20
-26
No files found.
scripts/_run_pytest.py
View file @
d540196f
# -*- coding: utf-8 -*-
""" NAME
_run_pytest -- private script
...
...
scripts/change_status_submitted.py
→
scripts/
archives_fix_py27/
change_status_submitted.py
View file @
d540196f
File moved
scripts/deploy.py
View file @
d540196f
# -*- coding: utf-8 -*-
""" NAME
deploy my_institute.csv
...
...
@@ -85,7 +84,7 @@ def create_affiliation(opt):
"""
aff
=
raw_input
(
"
\t
Enter the inspirehep identifier for your institute: "
)
if
not
aff
.
isdigit
():
print
"
\t\t
ERROR: invalid answer!"
print
(
"
\t\t
ERROR: invalid answer!"
)
# ........................................................................
#
...
...
@@ -104,12 +103,12 @@ def create_affiliation(opt):
is_key_add
=
False
for
key
in
keys
:
if
get_id
(
db
.
affiliation_keys
,
**
key
)
is
None
:
print
"
\t\t
Adding the affiliation keys '%s'"
%
key
[
"key_u"
]
print
(
"
\t\t
Adding the affiliation keys '%s'"
%
key
[
"key_u"
]
)
db
.
affiliation_keys
[
0
]
=
key
is_key_add
=
True
if
not
is_key_add
:
print
"
\t\t
Affiliation keys already exist!"
print
(
"
\t\t
Affiliation keys already exist!"
)
def
create_harvesters
(
opt
,
dfi
,
dfh
):
...
...
@@ -150,9 +149,9 @@ def create_harvesters(opt, dfi, dfh):
missings
.
append
(
elt
)
if
len
(
missings
)
>
0
:
print
"
\n\t
Please define the collaborations in data/harvesters.csv:"
print
(
"
\n\t
Please define the collaborations in data/harvesters.csv:"
)
for
elt
in
missings
:
print
"
\t\t
"
,
elt
print
(
"
\t\t
"
,
elt
)
print
sys
.
exit
(
1
)
...
...
@@ -163,8 +162,7 @@ def create_harvesters(opt, dfi, dfh):
#
recset
=
db
(
db
.
harvesters
.
id
>
0
)
if
recset
.
count
()
>
0
:
rep
=
\
raw_input
(
"
\n\t
Harvesters already exist. Remove them all [y/N]: "
)
rep
=
input
(
"
\n\t
Harvesters already exist. Remove them all [y/N]: "
)
if
rep
==
"y"
:
for
row
in
recset
.
iterselect
(
db
.
harvesters
.
id
):
...
...
@@ -177,7 +175,7 @@ def create_harvesters(opt, dfi, dfh):
for
key
,
value
in
CONTROLLER_TO_CAT
.
iteritems
():
id_category
=
get_id
(
db
.
categories
,
code
=
value
)
if
id_category
is
None
:
print
"
\n\t\t
ERROR: category '%s' is not defined. exit"
%
value
print
(
"
\n\t\t
ERROR: category '%s' is not defined. exit"
%
value
)
sys
.
exit
(
1
)
CONTROLLER_TO_CAT
[
key
]
=
id_category
...
...
@@ -195,12 +193,12 @@ def create_harvesters(opt, dfi, dfh):
id_team
=
get_id
(
db
.
teams
,
team
=
team
)
if
id_team
is
None
:
print
"
\n\t\t
Team '%s' is not defined. skip it!"
%
team
print
(
"
\n\t\t
Team '%s' is not defined. skip it!"
%
team
)
continue
id_project
=
get_id
(
db
.
projects
,
project
=
project
)
if
id_project
is
None
:
print
"
\n\t\t
Project '%s' is not defined. skip it!"
%
project
print
(
"
\n\t\t
Project '%s' is not defined. skip it!"
%
project
)
continue
data
=
dict
(
id_teams
=
id_team
,
id_projects
=
id_project
)
...
...
@@ -315,23 +313,23 @@ def create_teams_projects(opt, dfi):
#
id_team
=
get_id
(
db
.
teams
,
team
=
team
)
if
id_team
is
None
:
print
"
\n\t\t
Create team '%s'"
%
team
print
(
"
\n\t\t
Create team '%s'"
%
team
)
id_team
=
db
.
teams
.
insert
(
team
=
team
,
domain
=
row
.
domain
)
else
:
print
"
\n\t\t
team '%s' already exist!"
%
team
print
(
"
\n\t\t
team '%s' already exist!"
%
team
)
#
# project
#
id_project
=
get_id
(
db
.
projects
,
project
=
project
)
if
id_project
is
None
:
print
"
\t\t
Create project '%s'"
%
project
print
(
"
\t\t
Create project '%s'"
%
project
)
id_project
=
\
db
.
projects
.
insert
(
project
=
project
,
agencies
=
"CNRS/IN2P3"
)
else
:
print
"
\t\t
project '%s' already exist!"
%
project
print
(
"
\t\t
project '%s' already exist!"
%
project
)
#
# relation team / project
...
...
@@ -340,11 +338,11 @@ def create_teams_projects(opt, dfi):
get_id
(
db
.
organisation
,
id_teams
=
id_team
,
id_projects
=
id_project
)
if
rec_id
is
None
:
print
"
\t\t
Create the relation %s/%s"
%
(
team
,
project
)
print
(
"
\t\t
Create the relation %s/%s"
%
(
team
,
project
)
)
db
.
organisation
.
insert
(
id_teams
=
id_team
,
id_projects
=
id_project
)
else
:
print
"
\t\t
Relation %s/%s already exist!"
%
(
team
,
project
)
print
(
"
\t\t
Relation %s/%s already exist!"
%
(
team
,
project
)
)
def
harvesters
(
opt
):
...
...
@@ -362,7 +360,7 @@ def harvesters(opt):
- collection
"""
print
"
\t
Collect definitions of harvesters"
,
opt
.
path_harvesters
print
(
"
\t
Collect definitions of harvesters"
,
opt
.
path_harvesters
)
# absolute path in the docker container
path
=
os
.
path
.
join
(
os
.
getcwd
(),
...
...
@@ -372,7 +370,7 @@ def harvesters(opt):
opt
.
path_harvesters
)
if
not
os
.
path
.
exists
(
path
):
print
"
\n\t
File {0} don't exist!"
.
format
(
path
)
print
(
"
\n\t
File {0} don't exist!"
.
format
(
path
)
)
sys
.
exit
(
1
)
dh
=
(
pd
.
read_csv
(
path
)
...
...
@@ -398,17 +396,17 @@ def insert_harvester(**data):
id_harvester
=
get_id
(
db
.
harvesters
,
**
data
)
if
id_harvester
is
None
:
print
"
\n\t\t
Create harvester"
print
(
"
\n\t\t
Create harvester"
)
db
.
harvesters
.
insert
(
**
data
)
else
:
print
"
\n\t\t
Harvester exist!"
print
(
"
\n\t\t
Harvester exist!"
)
print
"
\t\t\t
team:"
,
db
.
teams
[
data
[
"id_teams"
]].
team
print
"
\t\t\t
project:"
,
db
.
projects
[
data
[
"id_projects"
]].
project
print
"
\t\t\t
controller:"
,
data
[
"controller"
]
print
"
\t\t\t
host:"
,
data
[
"host"
]
print
"
\t\t\t
collection:"
,
data
[
"collections"
]
print
(
"
\t\t\t
team:"
,
db
.
teams
[
data
[
"id_teams"
]].
team
)
print
(
"
\t\t\t
project:"
,
db
.
projects
[
data
[
"id_projects"
]].
project
)
print
(
"
\t\t\t
controller:"
,
data
[
"controller"
]
)
print
(
"
\t\t\t
host:"
,
data
[
"host"
]
)
print
(
"
\t\t\t
collection:"
,
data
[
"collections"
]
)
def
institute
(
opt
):
...
...
@@ -429,7 +427,7 @@ def institute(opt):
- hal_hidden
"""
print
"
\t
Collect parameters for the institute"
,
opt
.
institute
print
(
"
\t
Collect parameters for the institute"
,
opt
.
institute
)
# absolute path in the docker container
path
=
os
.
path
.
join
(
os
.
getcwd
(),
...
...
@@ -439,7 +437,7 @@ def institute(opt):
opt
.
institute
)
if
not
os
.
path
.
exists
(
path
):
print
"
\n\t
File {0} don't exist!"
.
format
(
path
)
print
(
"
\n\t
File {0} don't exist!"
.
format
(
path
)
)
sys
.
exit
(
1
)
df
=
pd
.
read_csv
(
path
)
...
...
@@ -511,11 +509,11 @@ if __name__ == "__main__":
OPT
=
PARSER
.
parse_args
()
if
not
(
OPT
.
harvesters
or
OPT
.
teams
or
OPT
.
affiliation
):
print
"
\n\t
Please select an option!"
print
"
\t
Possible ones are --affiliation, --harvesters or --teams"
print
(
"
\n\t
Please select an option!"
)
print
(
"
\t
Possible ones are --affiliation, --harvesters or --teams"
)
sys
.
exit
(
1
)
print
"
\n
%s
\n
Start deploy"
%
(
"."
*
80
,)
print
(
"
\n
%s
\n
Start deploy"
%
(
"."
*
80
,)
)
DFI
=
institute
(
OPT
)
DFH
=
harvesters
(
OPT
)
...
...
@@ -530,11 +528,11 @@ if __name__ == "__main__":
create_affiliation
(
OPT
)
if
OPT
.
commit
:
print
"
\n\t
Change are committed to the database."
print
(
"
\n\t
Change are committed to the database."
)
db
.
commit
()
else
:
print
"
\n\t
Dry run, the database is not modified."
print
(
"
\n\t
Dry run, the database is not modified."
)
print
"
\n
End deploy
\n
%s
\n
"
%
(
"."
*
80
,
)
print
(
"
\n
End deploy
\n
%s
\n
"
%
(
"."
*
80
,
)
)
sys
.
exit
(
0
)
scripts/export_to_csv.py
View file @
d540196f
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" NAME
export-to-csv -- export table content as CSV file
...
...
@@ -58,6 +56,6 @@ if __name__ == "__main__":
writer
.
writerow
(
row
.
as_dict
())
# close
print
'Table "%s" exported to "%s"'
%
(
args
.
table
,
args
.
file
.
name
)
print
(
'Table "%s" exported to "%s"'
%
(
args
.
table
,
args
.
file
.
name
)
)
args
.
file
.
close
()
sys
.
exit
(
0
)
scripts/import_from_csv.py
View file @
d540196f
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" NAME
import-from-csv -- import table content from CSV file
...
...
@@ -53,6 +51,6 @@ if __name__ == "__main__":
db
.
commit
()
# close
print
'File "%s" import in "%s"'
%
(
args
.
file
.
name
,
args
.
table
)
print
(
'File "%s" import in "%s"'
%
(
args
.
file
.
name
,
args
.
table
)
)
args
.
file
.
close
()
sys
.
exit
(
0
)
scripts/limbra_status.py
View file @
d540196f
# -*- coding: utf-8 -*-
""" NAME
status
...
...
@@ -71,7 +70,7 @@ if __name__ == "__main__":
for
el
in
db
().
iterselect
(
year
,
count
,
groupby
=
year
)}
if
len
(
publications
)
==
0
:
print
json
.
dumps
(
data
)
print
(
json
.
dumps
(
data
)
)
sys
.
exit
(
0
)
data
[
"publications"
]
=
publications
...
...
@@ -115,5 +114,5 @@ if __name__ == "__main__":
#
# output and exit
#
print
json
.
dumps
(
data
)
print
(
json
.
dumps
(
data
)
)
sys
.
exit
(
0
)
scripts/statistics.py
View file @
d540196f
# -*- coding: utf-8 -*-
""" NAME
statistics
...
...
@@ -24,6 +23,8 @@
R. Le Gac -- Mar 2015
"""
def
do_labels
(
ax
,
xlabel
,
ylabel
):
"""Deal with axes label.
...
...
@@ -109,11 +110,9 @@ def savefig(ax, fn):
if
__name__
==
"__main__"
:
import
matplotlib
as
mp
import
re
import
sys
from
argparse
import
ArgumentParser
,
FileType
from
invenio_tools
import
InvenioStore
,
Marc12
from
argparse
import
ArgumentParser
from
pandas
import
DataFrame
# command line options
...
...
@@ -132,17 +131,16 @@ if __name__ == "__main__":
q_base
=
db
.
publications
.
year
>=
start_year
q_base
=
(
q_base
)
&
(
db
.
publications
.
year
<=
end_year
)
#
--------------------------------------------------------------------------
#
........................................................................
#
# publications per domain and per year
#
#--------------------------------------------------------------------------
query
=
(
q_base
)
&
(
db
.
publications
.
id_teams
==
db
.
teams
.
id
)
rows
=
db
(
query
).
select
(
db
.
publications
.
year
,
db
.
teams
.
domain
,
count
,
groupby
=
(
db
.
publications
.
year
,
db
.
teams
.
domain
))
db
.
teams
.
domain
,
count
,
groupby
=
(
db
.
publications
.
year
,
db
.
teams
.
domain
))
data
=
[]
for
row
in
rows
:
...
...
@@ -163,24 +161,23 @@ if __name__ == "__main__":
kind
=
"bar"
,
stacked
=
True
,
xlabel
=
""
,
ylabel
=
u
'Nombre de publications / année'
)
ylabel
=
'Nombre de publications / année'
)
ax
.
set_xticklabels
(
list
(
df
.
index
),
rotation
=
0
)
fig
=
ax
.
get_figure
()
fig
.
show
()
#
--------------------------------------------------------------------------
#
........................................................................
#
# publications per category per year
#
#--------------------------------------------------------------------------
query
=
(
q_base
)
&
(
db
.
publications
.
id_categories
==
db
.
categories
.
id
)
rows
=
db
(
query
).
select
(
db
.
publications
.
year
,
db
.
categories
.
usual
,
count
,
groupby
=
(
db
.
publications
.
year
,
db
.
categories
.
usual
))
groupby
=
(
db
.
publications
.
year
,
db
.
categories
.
usual
))
data
=
[]
for
row
in
rows
:
...
...
@@ -197,7 +194,7 @@ if __name__ == "__main__":
df
.
columns
=
[
el
[
1
]
for
el
in
df
.
columns
]
# one column per year -- normalize the row on the sum per year
df
=
df
.
T
df
=
df
.
T
for
col
in
df
.
columns
:
df
[
col
]
=
df
[
col
]
/
df
[
col
].
sum
()
...
...
@@ -205,25 +202,24 @@ if __name__ == "__main__":
df
=
df
.
T
# prepare the stack by ordering the category (conference, article, report)
df
=
df
[[
0
,
5
,
1
,
2
,
3
,
4
,
6
]]
df
=
df
[[
0
,
5
,
1
,
2
,
3
,
4
,
6
]]
ax
=
do_plot
(
df
,
colormap
=
"Pastel1"
,
kind
=
"bar"
,
stacked
=
True
,
xlabel
=
""
,
ylabel
=
u
'Fraction / catégorie / année'
)
ylabel
=
'Fraction / catégorie / année'
)
ax
.
set_xticklabels
(
list
(
df
.
index
),
rotation
=
0
)
fig
=
ax
.
get_figure
()
fig
.
show
()
#
--------------------------------------------------------------------------
#
........................................................................
#
# Fraction of harvested publication per year
#
#--------------------------------------------------------------------------
query
=
q_base
rows
=
db
(
query
).
select
(
db
.
publications
.
year
,
...
...
@@ -253,7 +249,7 @@ if __name__ == "__main__":
kind
=
"bar"
,
stacked
=
True
,
xlabel
=
""
,
ylabel
=
u
'Fraction / entrepôt / année'
,
ylabel
=
'Fraction / entrepôt / année'
,
ylim
=
[
0
,
1.2
])
ax
.
set_xticklabels
(
list
(
df
.
index
),
rotation
=
0
)
...
...
@@ -261,11 +257,10 @@ if __name__ == "__main__":
fig
=
ax
.
get_figure
()
fig
.
show
()
#
--------------------------------------------------------------------------
#
........................................................................
#
# Fraction of harvested publication per category
#
#--------------------------------------------------------------------------
query
=
(
q_base
)
&
(
db
.
publications
.
id_categories
==
db
.
categories
.
id
)
rows
=
db
(
query
).
select
(
db
.
categories
.
usual
,
...
...
@@ -299,7 +294,7 @@ if __name__ == "__main__":
kind
=
"bar"
,
stacked
=
True
,
xlabel
=
""
,
ylabel
=
u
'Fraction / entrepôt / catégorie'
,
ylabel
=
'Fraction / entrepôt / catégorie'
,
ylim
=
[
0
,
1.2
])
ax
.
set_xticklabels
([
el
.
decode
(
'utf-8'
)
for
el
in
df
.
index
],
rotation
=
12
)
...
...
@@ -307,11 +302,10 @@ if __name__ == "__main__":
fig
=
ax
.
get_figure
()
fig
.
show
()
#
--------------------------------------------------------------------------
#
........................................................................
#
# Fraction of harvested publication per domain
#
#--------------------------------------------------------------------------
query
=
(
q_base
)
&
(
db
.
publications
.
id_teams
==
db
.
teams
.
id
)
rows
=
db
(
query
).
select
(
db
.
teams
.
domain
,
...
...
@@ -345,7 +339,7 @@ if __name__ == "__main__":
kind
=
"bar"
,
stacked
=
True
,
xlabel
=
""
,
ylabel
=
u
'Fraction / entrepôt / domaine'
,
ylabel
=
'Fraction / entrepôt / domaine'
,
ylim
=
[
0
,
1.2
])
ax
.
set_xticklabels
([
el
.
decode
(
'utf-8'
)
for
el
in
df
.
index
],
rotation
=
8
)
...
...
@@ -354,5 +348,5 @@ if __name__ == "__main__":
fig
.
show
()
# close
raw_
input
(
'type CR to exit'
)
input
(
'type CR to exit'
)
sys
.
exit
(
0
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment