Commit a055502d authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

Merge branch 'web_reshaping'

Conflicts:
	pipelet/__init__.py
	pipelet/db_utils.py
	pipelet/web.py
parents 4206ee5d 3dd474f9
......@@ -21,8 +21,7 @@ from task import *
from environment import *
__version__='a975fe4111c88190f726bf257cffa8237fbc3742'
__version__='a975fe4111c88190f726bf257cffa8237fbc3742'
__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
__version__='dc69c64a78bef2938ef4cfc27d1bc2d44d2616d8'
__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
......@@ -2,7 +2,11 @@ try:
import sqlite3
except ImportError:
import pysqlite2 as sqlite3
from glob import glob
import os
from contextlib import closing
import pickle
import shutil
def old_to_new (old_file, new_file):
""" Convert an old database to the new database format."""
conn1 = sqlite3.connect(old_file)
......@@ -33,6 +37,7 @@ def change_root(sql_file,old_prefix, new_prefix):
conn1.execute('update segments set curr_dir = replace(curr_dir, ?,?)', (old_prefix, new_prefix))
conn1.execute('update tasks set str_input = replace(str_input, ?,?)', (old_prefix, new_prefix))
def deltask(db_file, lst_task, report_only=False):
""" Delete a tasks instances.
......@@ -82,3 +87,264 @@ def deltask(db_file, lst_task, report_only=False):
lst_task.sort()
if lst_task:
deltask(db_file, lst_task, report_only=report_only)
def get_lst_tag (db_file):
""" Return the list of existing tags
Tags are ; separated in the db.
Returns
-------
list of string
"""
conn = sqlite3.connect(db_file,check_same_thread=True)
conn.text_factory=str
lst = []
with conn:
l = conn.execute('select tag from segments').fetchall()
for s in l:
if s[0] is not None:
str_tag = s[0].split(";")
for e in str_tag:
if e:
lst.append(e)
return lst
def get_lst_seg (db_file):
""" Return the list of segment name
Returns
-------
list of string
"""
conn = sqlite3.connect(db_file,check_same_thread=True)
conn.text_factory=str
lst = []
with conn:
l = conn.execute('select distinct seg from segments').fetchall()
lst = [s[0] for s in l]
return lst
def get_lst_date (db_file):
""" Return the list of existing dates
Date strings are picked from queued_on field corresponding to the
first task of each segment.
Returns
-------
list of string
"""
conn = sqlite3.connect(db_file,check_same_thread=True)
conn.text_factory=str
with conn:
l = conn.execute('select queued_on from tasks group by queued_on').fetchall()
return l[0]
def add_tag (db_file, segid, tag):
""" Add new tag to the database
Check that the tag do not exist yet for this id
Parameters
----------
segid: string. List of seg_id - separated. (- at the end of string also)
tag: string. Tags are ; separated in the db.
"""
seglst = segid.split('-')
conn = sqlite3.connect(db_file,check_same_thread=True)
conn.text_factory=str
for segid in seglst:
if segid:
with conn:
l = conn.execute('select tag,curr_dir from segments where seg_id=?',(segid,)).fetchone()
lst_tag = []
if l[0] is not None and l:
lst_tag = l[0].split(";")
lst_tag.append(tag)
lst_tag = list(set(lst_tag))
str_tag = ";".join(lst_tag)
conn.execute('update segments set tag=? where seg_id=?',(str_tag,segid))
fn = glob(os.path.join(l[1], "*.meta"))
_update_meta(fn, str_tag)
def _update_meta(fn, str_tag):
""" Update meta file with current tag value.
fn: string, meta file name
tag: string, tag string
"""
if fn:
with closing(file(fn[0], 'r')) as f:
d = pickle.load(f)
d['tag'] = str_tag
with closing(file(fn[0], 'w')) as f:
r = pickle.dump(d,f)
def del_tag (db_file, tag):
""" Delete tag from the database
Parameters
----------
tag: string.
"""
conn = sqlite3.connect(db_file,check_same_thread=True)
conn.text_factory=str
with conn:
l = conn.execute('select seg_id, tag, curr_dir from segments where tag like ?',("%"+tag+"%",)).fetchall()
for s in l:
lst_tag = s[1].split(";")
lst_tag.remove(tag)
str_tag = ";".join(lst_tag)
conn.execute('update segments set tag=? where seg_id=?',(str_tag,s[0]))
fn = glob(os.path.join(s[2], "*.meta"))
_update_meta(fn, str_tag)
def _delseg(db_file, lst_seg):
""" Delete a pipeline instance.
Delete all segments and products directories of a pipeline instance.
Parameters
----------
lst_seg: integer list
"""
## remove everyboby
conn = sqlite3.connect(db_file,check_same_thread=True)
segid = lst_seg[0]
with conn:
currdir = conn.execute('select curr_dir from segments where seg_id = ?',(int(segid),)).fetchone()[0]
l = conn.execute('select seg_id from segments where curr_dir like ?',(currdir+'%',))
for e in l:
lst_seg.remove(e[0])
## delete from tasks_relations
l = conn.execute('delete from task_relations where child_id in (select task_id from tasks where seg_id in (select seg_id from segments where segments.curr_dir like ?))',(currdir+'%',))
## delete from tasks
l = conn.execute('delete from tasks where seg_id in (select seg_id from segments where segments.curr_dir like ?)',(currdir+'%',))
## delete from segments_relations
l = conn.execute('delete from segment_relations where child_id in (select seg_id from segments where segments.curr_dir like ?)',(currdir+'%',))
## delete from segments
l = conn.execute('delete from segments where curr_dir like ?',(currdir+'%',))
conn.close()
try:
shutil.rmtree(currdir)
except:
pass
if lst_seg:
_delseg(db_file, lst_seg)
def _deltask(db_file, lst_task):
""" Delete a tasks instances.
Delete all products directories of a tasks instance.
Parameters
----------
lst_tag: string, list of pipe id '-' separated
"""
## remove everyboby
conn = sqlite3.connect(db_file,check_same_thread=True)
taskid = lst_task[0]
with conn:
str_input = conn.execute('select str_input from tasks where task_id = ?',(int(taskid),)).fetchone()[0]
lst_task.remove(taskid)
## delete from tasks_relations
l = conn.execute('delete from task_relations where child_id = ? ',(int(taskid),))
## delete from tasks
l = conn.execute('delete from tasks where task_id = ?',(int(taskid),))
conn.close()
try:
shutil.rmtree(str_input)
except:
pass
if lst_task:
_deltask(db_file, lst_task)
def _get_fathers(db_file, segid):
""" Append upstream pipeline instance paths to the current path.
Return a list which contains all upstream segment instances
id for a given segment instance. This is used to print a pipeline
tree view with all dependencies.
Parameters
----------
segid: id of the leaf segment.
Returns
-------
list of segid, for the upstream segment instances.
"""
lstid = [int(segid)]
conn = sqlite3.connect(db_file,check_same_thread=True)
with conn:
fids = conn.execute(
'select father_id from segment_relations where child_id = ?'
,(segid,)).fetchall()
conn.close()
if fids:
for l in fids:
lstid += _get_fathers(db_file,l[0])
return lstid
def _get_children(db_file, segid):
""" Return the list of pipeline instance paths which depend on seg
Return a list which contains all downstream segment instances
id for a given segment instance. This is used to delete all dependencies.
Parameters
----------
segid: id of the segment.
Returns
-------
list of segid, for the downstream segment instances.
"""
lstid = [int(segid)]
conn = sqlite3.connect(db_file,check_same_thread=True)
with conn:
fids = conn.execute(
'select child_id from segment_relations where father_id = ?'
,(segid,)).fetchall()
conn.close()
if fids:
for l in fids:
lstid += _get_children(db_file, l[0])
return lstid
def _get_children_task(db_file, taskid):
""" Return the list of tasks which depend on task.
Return a list which contains all downstream task instances
id for a given task instance. This is used to delete all dependencies.
Parameters
----------
taskid: id of the task
Returns
-------
list of taskid, for the downstream task instances.
"""
lstid = [int(taskid)]
conn = sqlite3.connect(db_file,check_same_thread=True)
with conn:
fids = conn.execute(
'select child_id from task_relations where father_id = ?'
,(taskid,)).fetchall()
conn.close()
if fids:
for l in fids:
lstid += _get_children_task(db_file, l[0])
return lstid
## Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr> <betoule@apc.univ-paris7.fr>
##
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, see http://www.gnu.org/licenses/gpl.html
try:
import sqlite3
except ImportError:
import pysqlite2 as sqlite3
import cPickle as pickle
import os
import glob
import cherrypy
import auth
from pipelet.web import TreeView
from copy import copy
class PBView(TreeView):
""" A pipeline Web interface dedicated to Polarbear experiment
It extends the minimalist tree view with a search object panel.
"""
def __init__ (self,db_file, name):
TreeView.__init__(self, db_file, name)
self.js_file += ["../static/pb.js"]
self.lst_ids = self.get_ids()
self.lst_mjd,self.lst_patch,self.lst_obs = self.get_mjd()
def get_ids (self):
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
with conn:
l1 = conn.execute('select seg_id,seg,param from segments where seg="stimtau"').fetchall()
l2 = conn.execute('select seg_id,seg,param from segments where seg="gatherstim"').fetchall()
conn.close()
ll = []
for e in l1+l2:
if e[2] is not None:#len(e[2])>41:
ll.append (e[1]+'@'+e[2][-41:])
else:
ll.append(e[1]+'@'+str(e[0]))
self.selected_id = l1[0][0]
self.selected_seg= l1[0][1]
self.selected_idtxt= ll[0]
return ll
def get_mjd (self):
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
ll = []
with conn:
l = conn.execute('select seg_id from segments where seg like "unpack%"').fetchall()
for s in l:
ll += conn.execute('select input,str_input from tasks where seg_id=? and status="done"',(s[0],)).fetchall()
conn.close()
lst_mjd = []
lst_patch= []
lst_dir = []
for e in ll:
i = pickle.loads(e[0])
lst_mjd += [k[0] for k in i.values()]
lst_patch+= [k[1] for k in i.values()]
tmplst = os.listdir(e[1])
lst_dir += [f for f in tmplst
if os.path.isdir(os.path.join(e[1],f)) ]
lst_mjd = list(set(lst_mjd))
lst_patch= list(set(lst_patch))
lst_dir = list(set(lst_dir))
return (lst_mjd,lst_patch, lst_dir)
@cherrypy.expose
@auth.pipeauth.read_access()
def get_folder (self, mjd):
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
segid = self.selected_id
with conn:
#l = conn.execute('select seg_id from segments where seg="stimtau"').fetchall()
#segid = l[0][0]
sqlr = 'select input,str_input from tasks where seg_id=%s and status="done" and input like "XXX%sXXX"'%(segid,mjd)
sqlr = sqlr.replace("XXX", "%")
l = conn.execute(sqlr).fetchall()
conn.close()
ll = ["stim"+str(pickle.loads(e[0]).values()[0][0]) for e in l]
self.lst_obs = ll
self.selected_mjd = mjd
raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
@cherrypy.expose
@auth.pipeauth.read_access()
def set_id (self, idtxt):
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
segid = self.selected_id
searchid = idtxt.split("@")[-1]
searchseg = idtxt.split("@")[0]
with conn:
sqlr = 'select seg_id,seg from segments where param like "XXX%sXXX" and seg="%s"'%(searchid[2:-1],searchseg)
sqlr = sqlr.replace("XXX", "%")
l = conn.execute(sqlr).fetchone()
conn.close()
self.selected_idtxt = idtxt
if idtxt not in self.lst_ids:
self.lst_ids.append(idtxt)
self.selected_id = l[0]
self.selected_seg = l[1]
raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
def filter_panel (self, l):
""" Return an html string containing the filter panel
"""
html = TreeView.filter_panel(self, l)
html += '<div id=right><fieldset id="right"><legend><span class="text">Search</span></legend>'
html += '<table width="100%">'
## Object chkbox
html += '<tr valign="top" id="tr_seg_id" class="filter"> <td style="width:150px;"><li><span class="smallcap">segment</span></li></td>'
##<input id="cb_seg" type="checkbox"/><label></label></td>'
## Object select
html += '<td style="width:50px;"><select id="se_seg_id" onchange=javascript:updateid();>'
for t in self.lst_ids:
if t==self.selected_idtxt:
html +='<option selected value="%s">%s</option>'%(self.selected_idtxt,self.selected_idtxt)
else:
html +='<option value="%s">%s</option>'%(t, t)
html += '</select></td>'
## Input chkbox
dd = dict({"mjd":(self.lst_mjd,"mjd"), "patch":(self.lst_patch,"patchname"), "folder":(self.lst_obs, "folder")})
for k,v in dd.iteritems():
html += '<tr valign="top" id="tr_id" class="filter"> <td style="width:150px;"><li><span class="smallcap">%s</span></li></td>'%(v[1])
##<input id="cb_%s" type="checkbox"/><label>%s</label></td>'%(k,v[1])
if k=='mjd':
html += '<td style="width:50px;"><select id="se_%s_id" onchange=javascript:updatemjd();>'%k
else:
html += '<td style="width:50px;"><select id="se_%s_id">'%k
vlist = copy(v[0])
if hasattr(self, 'selected_mjd') and k=="mjd":
html +='<option selected value="%s">%s</option>'%(self.selected_mjd, self.selected_mjd)
vlist.remove(self.selected_mjd)
for t in vlist:#v[0]:
html +='<option value="%s">%s</option>'%(t, t)
html += '</select></td>'
## Buttons
html += '<td class="buttons">'
html += '<a class="icon apply" href="javascript:ssearch();"><small>Apply</small></a>'
html += '</td><tr></tr></table>'
html += '</fieldset></div>'
return html
@cherrypy.expose
@auth.pipeauth.read_access()
def search(self , mjd=None, folder=None, patch=None,obj=None):
"""
"""
import cPickle as pickle
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
if self.selected_seg=='stimtau':#folder != "None" and mjd != "None":
foldernum = int(folder.replace("stim", ""))
with conn:
l = conn.execute('select seg_id,curr_dir from segments where seg_id=?',(self.selected_id,)).fetchall()
segid = l[0][0]
currdir = l[0][1]
sqlr = 'select str_input,task_id from tasks where seg_id="%s" and status="done" and input like "XXX%sXXX%sXXX"'%(segid,pickle.dumps(foldernum)[:-1], mjd)
sqlr = sqlr.replace("XXX", "%")
l = conn.execute(sqlr).fetchall()
conn.close()
r = os.path.relpath(l[0][0], currdir)
directory = r.replace("../",'')
html = self.pipedir(segid=segid, directory=directory)
# elif folder =="None" and mjd != "None":
# with conn:
# l = conn.execute('select seg_id,curr_dir from segments where seg=?',("unpack_obs_stimulator",)).fetchall()
# segid = l[0][0]
# currdir = l[0][1]
# sqlr = 'select str_input,task_id from tasks where seg_id="%s" and status="done" and input like "XXX%sXXX"'%(segid,mjd)
# sqlr = sqlr.replace("XXX", "%")
# l = conn.execute(sqlr).fetchall()
# conn.close()
# r = os.path.relpath(l[0][0], currdir)
# directory = r.replace("../",'')
# html = self.pipedir(segid=segid, directory=directory)
elif self.selected_seg=='gatherstim':
with conn:
l = conn.execute('select seg_id,curr_dir from segments where seg_id=?',(self.selected_id,)).fetchall()
segid = l[0][0]
currdir = l[0][1]
sqlr = 'select str_input from tasks where seg_id="%s" and status="done"'%(segid)
l = conn.execute(sqlr).fetchall()
conn.close()
r = os.path.relpath(l[0][0], currdir)
directory = r.replace("../",'')
html = self.pipedir(segid=segid, directory=directory)
return html
......@@ -87,6 +87,20 @@ a{
}
a:hover { text-decoration: underline; }
#right{
float:left;
width:55%;
}
#left{
float:left;
width:45%;
}
#thumb{
width:95%;
}
#pipelist{
width:60%;
}
......@@ -127,4 +141,10 @@ h1{
/* tag display style */
.pipetag{
text-shadow:2px 2px #FF0000;
}
.smallcap{
font-weight: bold;
font-family:Helvetica,Arial,sans-serif;
font-variant: small-caps
}
\ No newline at end of file
// Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr> <betoule@apc.univ-paris7.fr>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, see http://www.gnu.org/licenses/gpl.html
// filter pipelines
function ssearch(){
//var cb1 = document.getElementById('cb_mjd') ;
//var is1 = cb1.checked;
var mjd = "None";
//if (is1)
//{
var seltag = document.getElementById('se_mjd_id') ;
mjd = seltag.options[seltag.selectedIndex].text;
//}
//var cb2 = document.getElementById('cb_folder') ;
//var is2 = cb2.checked;
var folder = "None";
//if (is2)
//{
var seldate = document.getElementById('se_folder_id') ;
folder = seldate.options[seldate.selectedIndex].text;
//}
var url = "search?mjd="+mjd+"&folder="+folder;
//var url = "blala";
self.location.href=url;
}
function updatemjd () {
var seltag = document.getElementById('se_mjd_id') ;
var mjd = seltag.options[seltag.selectedIndex].text;
self.location.href="get_folder?mjd="+mjd;
}
function updateid () {
var seltag = document.getElementById('se_seg_id') ;
var seg = seltag.options[seltag.selectedIndex].text;
self.location.href="set_id?idtxt="+seg;
}
......@@ -91,7 +91,6 @@ function del_prod(segid) {
}
self.location.href=url;
}
// filter pipelines
function filter(){
......@@ -114,3 +113,4 @@ function filter(){
var url = "filter?tag="+tag+"&date="+date;
self.location.href=url;
}