Commit fc2ae2d1 authored by LE GAC Renaud's avatar LE GAC Renaud
Browse files

Merge branch 'master' into 'production'

Release 0.9.5.3

* Minor release
* Move to plugin_dbui 0.8.2.
* Require the rpms `libreoffice-headless`.
* Series of modification to run via a docker image.
* `build_version.py` can install plugins.

See merge request !62
parents 4fc172d1 6f93a780
0.9.5.3
\ No newline at end of file
......@@ -2,14 +2,17 @@
# -*- coding: utf-8 -*-
"""
NAME
build_version -- build and tag a track_publications version
build_version -- set of tool to build release
SYNOPSIS
build_version [options]
DESCRIPTION
Helper script to build a version of the track_publications.
Push version number in the CHANGELOG.
Helper script to build release and to push version number
in CHANGELOG and VERSION files.
It also performed the installation of the plugins, generated
the documentation and minificated javascript libraries.
EXAMPLES
......@@ -18,7 +21,7 @@
AUTHOR
R. Le Gac, renaud.legac@cppm.in2p3.fr
Copyright (c) 2012-2015 R. Le Gac
Copyright (c) 2012-2016 R. Le Gac
"""
import argparse
......@@ -29,34 +32,31 @@ import sys
import tempfile
from glob import glob
from os.path import join as opj
from subprocess import call
from subprocess import call, check_output
CWD = os.getcwd()
# constants
APP = os.path.basename(os.getcwd())
API = 'api'
CHANGELOG = 'static/CHANGELOG'
DBUISRC = 'static/plugin_dbui/src'
DOCS = 'static/docs'
CHANGELOG = os.path.join(CWD, 'static/CHANGELOG')
DBUISRC = os.path.join(CWD, 'static/plugin_dbui/src')
DOCS = os.path.join(CWD, 'static/docs')
DOCSRC = 'docs'
EXTJSSRC = 'static/plugin_extjs/src'
EXTJSSRC = os.path.join(CWD, 'static/plugin_extjs/src')
JSDOC = opj(DOCS, 'jsduck')
JSLIBDEBUG = 'static/%s-debug.js' % APP
JSLIBMIN = 'static/%s-min.js' % APP
JSLIBSRC = 'static/%s/src' % APP
JSLIBDEBUG = os.path.join(CWD, 'static/%s-debug.js' % APP)
JSLIBMIN = os.path.join(CWD, 'static/%s-min.js' % APP)
JSLIBSRC = os.path.join(CWD, 'static/%s/src' % APP)
LATEX = 'latex'
NOW = datetime.datetime.now()
PDF = "pdf"
PDFDOC = opj(DOCS, PDF)
REFERENCE = 'reference'
USER = 'user'
# basic commands
GIT = '/usr/bin/git'
JSDUCK = os.path.expandvars("$HOME/bin/jsduck")
PDFLATEX = '/usr/bin/pdflatex'
SENCHA = os.path.expandvars("$HOME/lib/Sencha/Cmd/latest/sencha")
SPHINX = '/usr/bin/sphinx-build'
VERSION = 'VERSION'
MSG_RELEASE = 'Enter the new release: '
......@@ -105,6 +105,11 @@ def build_pdf(doc):
"""
print "Build the PDF documentations..."
if not which("pdflatex"):
print '\n\tThe application pdflatex is missing !'
print '\tSkip this step.\n'
return
latexdoc = opj(DOCS, LATEX)
# generate the latex
......@@ -123,8 +128,8 @@ def build_pdf(doc):
return
# process the latex file twice
call([PDFLATEX, filename])
call([PDFLATEX, filename])
call(["pdflatex", filename])
call(["pdflatex", filename])
# move the pdf file
os.chdir(cwd)
......@@ -144,18 +149,18 @@ def build_pdf(doc):
def change_log():
"""Commit CHANGELOG.
"""Commit CHANGELOG and VERSION files
"""
print "Commit CHANGELOG..."
if not os.path.exists(GIT):
if not which("git"):
print '\n\tThe application git is missing !'
print '\tSkip this step.\n'
return
# Commit modified files
print 'git add', CHANGELOG
git("add", CHANGELOG)
print 'git add', CHANGELOG, VERSION
git("add", CHANGELOG, VERSION)
print 'git commit'
msg = "Start release %s" % get_version()
......@@ -186,7 +191,7 @@ def compile_js():
"""
print "Compile the javascript code ..."
if not os.path.exists(SENCHA):
if not which("sencha"):
print '\n\tThe application sencha is missing !'
print '\tSee: http://www.sencha.com/products/sencha-cmd/download'
print '\tSkip this step.\n'
......@@ -224,19 +229,14 @@ def compile_js():
def get_version():
"""Get the current version identifier in the CHANGELOG
"""Get the current version identifier in VERSION
Returns:
str: the version number
"""
log = open(CHANGELOG).read()
match = re.search(r"(\d+\.\d+(\.\d+)*)", log)
if match:
return match.group(1)
return ""
with open(VERSION, "rb") as fi:
return fi.read()
def git(*args, **kwargs):
......@@ -254,12 +254,12 @@ def git(*args, **kwargs):
git("commit", stdout=fi)
"""
if not os.path.exists(GIT):
if not which("git"):
print '\n\tThe application git is missing !'
print '\tSkip this step.\n'
return
cmd = [GIT]
cmd = ["git"]
cmd.extend(args[0].split())
if len(args) > 1:
......@@ -268,13 +268,132 @@ def git(*args, **kwargs):
call(cmd, **kwargs)
def install_plugins(args):
"""Install or update plugins.
Args:
args (argparse.Namespace): command arguments
"""
if args.plugins_tar:
install_plugins_tar(args.plugins_tar, args.plugins)
elif args.plugins_git:
for plugin in args.plugins:
install_plugin_git(args.plugins_git, plugin, args.plugins_release)
else:
print "Path to the plugins can be either a git repository",
print "or a tar file. One of them has to be defined !"
sys.exit(1)
def install_plugin_git(repository, plugin, release):
"""Install plugin from git repository
Args:
repository (str): path of the git repository.
plugin (str): name of the plugin to be installed.
release (str): release number for plugins.
"""
print "\nInstall plugin", plugin
# delete existing version
for path in glob("./*/plugin_%s*" % plugin):
print "Remove", path
call(["rm", "-rf", path])
# select the release
app_dir = os.getcwd()
os.chdir(repository)
git_dir = os.getcwd()
tag = ("master" if release == "latest" else release)
git("checkout", tag)
pattern = os.path.join(git_dir, "web2py.plugin.%s.*.w2p" % plugin)
paths = glob(pattern)
# select the compact version when possible
if len(paths) == 0:
print "No plugin", plugin
sys.exit(1)
elif len(paths) == 2:
for el in paths:
if el.endswith("min.w2p"):
path = el
break
else:
path = paths[0]
# install
print "tar", "-xf", path, "-C", app_dir
call(["tar", "-xf", path, "-C", app_dir])
# restore the git repository
git("checkout", "master")
os.chdir(app_dir)
def install_plugins_tar(path, plugins):
"""Install plugin from tar file
Args:
path (str): path of the tar file.
plugins (list): list of plugin to be installed.
"""
app_dir = os.getcwd()
# untar in /tmp
tar_dir = '/tmp/w2p_plugins'
if not os.path.exists(tar_dir):
os.mkdir(tar_dir)
call(["tar", "-xf", path, "-C", tar_dir])
# untar individual files
for plugin in plugins:
# delete existing version
for path in glob("./*/plugin_%s*" % plugin):
print "Remove", path
call(["rm", "-rf", path])
# get the plugin path
pattern = os.path.join(tar_dir, "web2py.plugin.%s.*.w2p" % plugin)
paths = glob(pattern)
# select the compact version when possible
if len(paths) == 0:
print "No plugin", plugin
sys.exit(1)
elif len(paths) == 2:
for el in paths:
if el.endswith("min.w2p"):
path = el
break
else:
path = paths[0]
# install
print "tar", "-xf", path, "-C", app_dir
call(["tar", "-xf", path, "-C", app_dir])
# cleam
call("cd /tmp && rm -rf w2p_plugins", shell=True)
def jsduck():
"""Generate the JavaScript documentation.
The HTML files are located in static/plugin_dbui/docs/jsduck
"""
print "Build the javascript documentation..."
if not os.path.exists(JSDUCK):
if not which("jsduck"):
print '\n\tThe application jsduck is missing !'
print '\tSkip this step.\n'
return
......@@ -292,9 +411,9 @@ def jsduck():
call(cmd)
# run JsDuck
cmd = ["jsduck", EXTJSSRC, JSLIBSRC, \
"--output", JSDOC, \
"--title", "plugin_dbui %s" % get_version(), \
cmd = ["jsduck", EXTJSSRC, JSLIBSRC,
"--output", JSDOC,
"--title", "plugin_dbui %s" % get_version(),
"--warnings=-all:" + EXTJSSRC]
call(cmd)
......@@ -303,7 +422,7 @@ def jsduck():
def set_version(version):
"""Set release identifier in CHANGELOG.
"""Set release identifier in CHANGELOG and VERSION.
Args:
version (str): release identifier
......@@ -312,22 +431,19 @@ def set_version(version):
print "Update CHANGELOG with the release", version, "..."
# check tag in git
fichier = tempfile.TemporaryFile()
git("tag", stdout=fichier)
fichier.seek(0)
if version in fichier.read():
if version in check_output(["git", "tag"]):
print "\n\tRelease %s already exit in git" % version
sys.exit(1)
# look for a pattern HEAD in the CHANGELOG
# split the the string in 2 parts (pre HEAD, post HEAD)
print 'Set release', version, 'in', CHANGELOG
txt = open(CHANGELOG, 'rb').read()
with open(CHANGELOG) as fi:
txt = fi.read()
match = re.match("(.+HEAD\n)(.*)", txt, re.DOTALL)
if match == None:
if match is None:
print '\n\tNo HEAD tag in the CHANGELOG!\n'
rep = raw_input('\tDo you want to continue [n]?')
if rep not in ('y', 'yes'):
......@@ -336,9 +452,10 @@ def set_version(version):
# update the version and edit the CHANGELOG
tpl = (match.group(1), version, NOW.strftime('%b %Y'), match.group(2))
txt = '%s\n%s (%s)\n%s' % tpl
fichier = open(CHANGELOG, 'wb')
fichier.write(txt)
fichier.close()
with open(CHANGELOG, 'w') as fi:
fi.write(txt)
call(["vim", CHANGELOG])
# cleaning
......@@ -346,6 +463,10 @@ def set_version(version):
if os.path.exists(filename):
os.remove(filename)
# update VERSION
with open(VERSION, 'w') as fichier:
fichier.write(version)
def sphinx(*args, **kwargs):
"""run the sphinx-build:
......@@ -361,12 +482,12 @@ def sphinx(*args, **kwargs):
sphinx("-b html", sourcedir, outdir)
"""
if not os.path.exists(SPHINX):
if not which("sphinx-build"):
print '\n\tThe application sphinx is missing !'
print '\tSkip this step.\n'
return
cmd = [SPHINX]
cmd = ["sphinx-build"]
cmd.extend(args[0].split())
if len(args) > 1:
......@@ -395,6 +516,24 @@ def start_release():
build()
def which(cmd):
"""Test if the command exists on the system.
Args:
name (str): name of the command
Returns:
bool:
"""
try:
check_output(["which", cmd])
return True
except CalledProcessError:
return False
if __name__ == '__main__':
# define script options
......@@ -410,7 +549,7 @@ if __name__ == '__main__':
APS.add_argument("-b", "--build",
action="store_true",
help="build the javascript library and the documentation.")
help="build the js library and the documentation.")
APS.add_argument("-c", "--compile",
action="store_true",
......@@ -456,10 +595,32 @@ if __name__ == '__main__':
APS.add_argument("-w", "--write-release",
action="store_true",
help="write the release number in CHANGELOG. "
help="write the release number in CHANGELOG and VERSION. "
"To be used with --commit-changelog. "
"Recommend to use --start-release.")
APS.add_argument("--plugins",
nargs="+",
help="plugins to be installed. To be used with "
"--plugin-release and --plugin-git or --plugin-tar."
" Possible values are ace, dbui, extjs and mathjax.",
metavar="<plugin>")
APS.add_argument("--plugins-release",
default="latest",
help="release number for plugin [%(default)s]. "
"To be used with --plugin-git.",
metavar="<release>")
APS.add_argument("--plugins-git",
default="../plugin_dbui_build",
help="git repository containing plugins [%(default)s].",
metavar="<path>")
APS.add_argument("--plugins-tar",
help="tar file containing the plugins",
metavar="<path>")
ARGS = APS.parse_args()
if ARGS.api_doc:
......@@ -480,6 +641,9 @@ if __name__ == '__main__':
if ARGS.jsduck:
jsduck()
if ARGS.plugins:
install_plugins(ARGS)
if ARGS.write_release:
set_version(raw_input(MSG_RELEASE))
......
......@@ -12,11 +12,11 @@
# serve to show the default.
import os
import re
import sys
from datetime import datetime
def get_version():
"""Get the current version identifier.
......@@ -24,20 +24,22 @@ def get_version():
str: the version number
"""
log = open('../../static/CHANGELOG').read()
match = re.search(r"(\d+\.\d+(\.\d+)*)", log)
if match:
return match.group(1)
return ""
with open('../../VERSION') as fi:
return fi.read()
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('../../modules'))
sys.path.insert(0, os.path.abspath('../../../web2py'))
# docker image
if os.path.exists("/opt/web2py"):
sys.path.insert(0, os.path.abspath('/opt/web2py'))
# older running environment
else:
sys.path.insert(0, os.path.abspath('../../../web2py'))
# -- General configuration -----------------------------------------------------
......
......@@ -12,13 +12,6 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
import re
import shlex
import tempfile
from datetime import datetime
from subprocess import call
......@@ -30,13 +23,8 @@ def get_version():
str: the version number
"""
log = open('../../static/CHANGELOG').read()
match = re.search(r"(\d+\.\d+(\.\d+)*)", log)
if match:
return match.group(1)
return ""
with open('../../VERSION') as fi:
return fi.read()
def git(*args, **kwargs):
......
......@@ -406,6 +406,7 @@
'Number of records validated: %s': "Nombre d'enregistrements validé : %s",
'Number(s)': 'Numéro(s)',
'Object or table name': 'Object or table name',
'ODT conversion requires an X11 server.': "La conversion ODT a besoin d'un serveur X11.",
'Ok': 'Ok',
'Old password': 'Old password',
'Open source': 'Code open source',
......
......@@ -5,7 +5,6 @@
import datetime
import filters
import harvest_tools
import locale
import plugin_dbui as dbui
from callbacks import (INHIBIT_CASCADE_DELETE,
......@@ -46,8 +45,6 @@ T.set_current_languages('en', 'en-gb', 'en-us') # mother tongue
T.force('fr-fr') # user language
T.lazy = False # immediate translation
locale.setlocale(locale.LC_ALL, "fr_FR.UTF-8")
#-------------------------------------------------------------------------------
#
# CONSTANTS
......
......@@ -35,26 +35,16 @@ def do_empty(db, selector, target=None):
year_end = selector.year_end
if year_start and not year_end:
nyear = 1
year = int(year_start)
months = to_datetime(["%i-%02i" % (year, i) for i in xrange(1, 13)])
dates = to_datetime(["%s-01-01" % year_start, "%s-12-31" % year_start])
elif year_start and year_end:
year_start = int(year_start)
year_end = int(year_end)
nyear = year_end - year_start + 1
li = []
for year in xrange(int(year_start), int(year_end)+1):
li.extend(["%i-%02i" % (year, i) for i in xrange(1, 13)])
months = to_datetime(li)
dates = to_datetime(["%s-01-01" % year_start, "%s-12-31" % year_end])
else:
nyear = 1
year = datetime.now().year
months = to_datetime(["%i-%02i" % (year, i) for i in xrange(1, 13)])
dates = to_datetime(["%i-01-01" % year, "%i-12-31" % year])
df = DataFrame([0]*12*nyear, index=months)
df = DataFrame([0]*2, index=dates)
ax = df.plot(legend=False, x_compat=True, ax=target)
do_tick(ax)
......
......@@ -94,7 +94,8 @@ if __name__ == "__main__":
data["year"] = match.group(offset+3)
month = MONTHS[match.group(offset+2)]
data["submitted"] = "%s-%s-%s" % (match.group(offset+3), month, match.group(offset+1))
data["submitted"] = \
"%s-%02i-%02i" % (match.group(offset+3), int(month), int(match.group(offset+1)))
else:
print "No conferences dates", data["id"]
......
......@@ -2,6 +2,11 @@
HEAD
0.9.5.3 (Feb 2016)
- Move to plugin_dbui 0.8.2.
- Series of modification to run via a docker image.
- build_version.py can install plugins.
0.9.5.2 (Jan 2016)
- Fix a bug in the controller edit_and_insert.
- Fix a bug in the function duplicate_conference.
......
......@@ -2,7 +2,6 @@
"""test harvest_tools functions
"""
import locale
import pytest
......@@ -10,10 +9,6 @@ from gluon import current
from harvest_tools import format_author_fr, search_synonym, ToolException
from invenio_tools import load_record
def test_locale():