Commit 9f359413 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Add two scripts (export-to-csv, import-from-csv) and remove the obsolete toolbox controller.

parent 90f47f75
""" Tool box to ease the maintenance/migration of the database.
"""
import csv
import os
import re
from invenio_tools import MONTHS
from plugin_dbui import get_id, get_where_query
def change_publisher():
"""Review attached to publications
../track_publications/toolbox/change_publisher/57/2
or
../track_publications/toolbox/change_publisher/57
to count
publishers id 57 will be replaced by 2
"""
args = request.args
query = get_where_query(db.publications)
query = ((query) & (db.publications.id_publishers == int(args[0])))
if len(args) == 1:
return "%s → %i" % (args[0], db(query).count())
elif len(args) == 2:
for el in db(query).select():
print el.publications.id, el.publications.id_publishers, int(args[1])
db.publications[el.publications.id] = dict(id_publishers = int(args[1]))
else:
return '???'
def doublon():
"""Remove duplicate entries for a team, a year and a categories
../track_publication/toolbox/doublon/Atals/2012
"""
if len(request.args) != 2:
return 'invalid → track_publication/toolbox/doublon/Atlas/2012'
team, year = request.args[0], int(request.args[1])
print team, year
query = get_where_query(db.publications)
query = ((query) & (db.teams.team == team))
query = ((query) & (db.categories.code == 'ACL'))
query = ((query) & (db.publications.year == year))
for el in db(query).select():
q = ((query) & (db.publications.title == el.publications.title))
q = ((q) & (db.publications.volume == el.publications.volume))
q = ((q) & (db.publications.pages == el.publications.pages))
if db(q).count() == 2:
ids, id_publishers = [], []
for row in db(q).select():
ids.append(row.publications.id)
id_publishers.append(row.publications.id_publishers)
print row.publications.id,
print row.publications.title,
print row.publications.id_collaborations,
print row.publications.id_teams,
print row.publications.id_projects,
print row.publications.id_publishers,
# print row.publishers.abbreviation,
print row.publications.volume,
print row.publications.pages
if id_publishers[0] != id_publishers[1]:
print 'delete', ids[1]
# del db.publications[ids[1]]
else:
print 'delete', ids[1]
del db.publications[ids[1]]
print
return 'game over'
#@auth.requires_login()
def export_csv():
"""export a table as csv file.
The action is steered via the URL:
http://localhost:8000/track_publications/toolbox/export_csv?table=foo&fn=foo.csv
The csv file is located in the web2py main directory.
"""
if 'table' not in request.vars:
return 'Specify the name of the table !!!'
tablename = request.vars.table
if tablename not in db.tables:
return 'Invalid database table name "%s"' % tablename
if 'fn' not in request.vars:
return 'Specify the name of the csv file !!!'
fn = request.vars.fn
fi = open(fn, 'w')
writer = csv.DictWriter(fi,
db[tablename].fields,
quoting=csv.QUOTE_MINIMAL)
# write the header
headers = {}
for el in db[tablename].fields:
headers[el] = el
writer.writerow(headers)
# write rows
for row in db(db[tablename]).select():
writer.writerow(row.as_dict())
# close and exit
fi.close()
return 'Table "%s" export in "%s"' % (tablename, fn)
def fix_none():
"""Replace None by the field defalut value.
The action is steered via the URL:
http://localhost:8000/track_publications/toolbox/fix_none
"""
for table in db:
if table._tablename.startswith("auth_"):
continue
print table._tablename
for row in db().select(table.ALL):
id, di = row.id, {}
print " "*2, row.id
for key in row:
if row[key] is None:
if key.startswith('id'):
print " "*4, "?"*4, key, row[key]
continue
print " "*4, key, row[key], ">"+table[key].default+"<"
di[key] = table[key].default
# if di:
# table[id] = di
return 'Game over'
#@auth.requires_login()
def import_csv():
"""import a csv file in a database table.
The csv file is produced by using the export_csv method.
This method preserved id for the primary and the foreign keys.
The action is steered via the URL:
http://localhost:8000/track_publications/toolbox/import_csv?table=foo&fn=foo.csv
The csv file is located in the web2py main directory.
"""
if 'table' not in request.vars:
return 'Specify the name of the table !!!'
tablename = request.vars.table
if tablename not in db.tables:
return 'Invalid database table name "%s"' % tablename
if 'fn' not in request.vars:
return 'Specify the name of the csv file !!!'
fn = request.vars.fn
fi = open(fn, 'r')
reader = csv.DictReader(fi)
for di in reader:
db[tablename][0] = di
return 'File "%s" import in "%s"' % (fn, tablename)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" NAME
export-to-csv -- export table content as CSV file
SYNOPSIS
export-to-csv [options] table file
DESCRIPTION
Write the full content of the table to a CSV file.
OPTIONS
-h, --help
Display the help and exit.
EXAMPLE
> cd ...track_publications/scripts
> ./track_publications export-to-csv.py publications ~/mywap/track_publications/scripts/publications.csv
AUTHOR
R. Le Gac
"""
if __name__ == "__main__":
import csv
import os
import sys
from argparse import ArgumentParser, FileType
# command line options
parser = ArgumentParser()
parser.add_argument('table',
help='the name of the table.')
parser.add_argument('file',
type=FileType('w'),
help='the absolute path of the CSV file.')
args = parser.parse_args()
# the CSV writer
writer = csv.DictWriter(args.file,
db[args.table].fields,
quoting=csv.QUOTE_MINIMAL)
# write the header
headers = {}
for el in db[args.table].fields:
headers[el] = el
writer.writerow(headers)
# write table content
for row in db(db[args.table]).select():
writer.writerow(row.as_dict())
# close
print 'Table "%s" exported to "%s"' % (args.table, args.file.name)
args.file.close()
sys.exit(0)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" NAME
import-from-csv -- import table content from CSV file
SYNOPSIS
import-from-csv [options] file table
DESCRIPTION
Load the full content of the table from a CSV file.
OPTIONS
-h, --help
Display the help and exit.
EXAMPLE
> cd ...track_publications/scripts
> ./track_publications import-from-csv.py ~/mywap/track_publications/scripts/publications.csv publications
AUTHOR
R. Le Gac
"""
if __name__ == "__main__":
import csv
import os
import sys
from argparse import ArgumentParser, FileType
TRACK_PUBLICATIONS_DIR = '../track_publications'
# command line options
parser = ArgumentParser()
parser.add_argument('file',
type=FileType('r'),
help='the absolute path of the CSV file.')
parser.add_argument('table',
help='the name of the table.')
args = parser.parse_args()
# load table content
reader = csv.DictReader(args.file)
for di in reader:
db[args.table][0] = di
db.commit()
# close
print 'File "%s" import in "%s"' % (args.file.name, args.table)
args.file.close()
sys.exit(0)
......@@ -16,6 +16,8 @@ HEAD
- Refactor the ux files to ease the buiding of a new database from scratch.
- Use the new callbacks INHIBIT_DELETE_UNDEF and INHIBIT_UPDATE_UNDEF.
- Redesing the lists and metrics interfaces.
- Remove obsolete controller toolbox. It is now replaced by standalone
scripts: export-to-csv and import-from-csv.
- The list of country is almost frozen. Add the script fix-country.py
to correct invalid country name in the exiting database.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment