Merge branch 'web_reshaping'

Conflicts:
	pipelet/__init__.py
	pipelet/db_utils.py
	pipelet/web.py

pipelet/__init__.py

@@ -21,8 +21,7 @@ from task import *
 from environment import *
 
 
-__version__='a975fe4111c88190f726bf257cffa8237fbc3742'
 
-__version__='a975fe4111c88190f726bf257cffa8237fbc3742'
+__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
 
-__version__='dc69c64a78bef2938ef4cfc27d1bc2d44d2616d8'
+__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'

pipelet/db_utils.py

@@ -2,7 +2,11 @@ try:
     import sqlite3
 except ImportError:
     import pysqlite2 as sqlite3
-
+from glob import glob
+import os
+from contextlib import closing
+import pickle
+import shutil
 def old_to_new (old_file, new_file):
     """ Convert an old database to the new database format."""
     conn1 = sqlite3.connect(old_file)
@@ -33,6 +37,7 @@ def change_root(sql_file,old_prefix, new_prefix):
         conn1.execute('update segments set curr_dir = replace(curr_dir, ?,?)', (old_prefix, new_prefix))
         conn1.execute('update tasks set str_input = replace(str_input, ?,?)', (old_prefix, new_prefix))
 
+
 def deltask(db_file, lst_task, report_only=False):
     """ Delete a tasks instances.
 
@@ -82,3 +87,264 @@ def deltask(db_file, lst_task, report_only=False):
         lst_task.sort()
     if lst_task:
         deltask(db_file, lst_task, report_only=report_only)
+
+def get_lst_tag (db_file):
+    """ Return the list of existing tags
+    
+    Tags are ; separated in the db. 
+    
+    Returns
+    -------
+    list of string
+    """
+    conn = sqlite3.connect(db_file,check_same_thread=True)
+    conn.text_factory=str
+    lst = []
+    with conn:
+        l = conn.execute('select tag from segments').fetchall()
+        for s in l:
+            if s[0] is not None:
+                str_tag = s[0].split(";")
+                for e in str_tag:
+                    if e:
+                        lst.append(e)
+    return lst
+
+def get_lst_seg (db_file):
+    """ Return the list of segment name
+    
+    Returns
+    -------
+    list of string
+    """
+    conn = sqlite3.connect(db_file,check_same_thread=True)
+    conn.text_factory=str
+    lst = []
+    with conn:
+        l = conn.execute('select distinct seg from segments').fetchall()
+        lst = [s[0] for s in l]
+    return lst
+
+def get_lst_date (db_file):
+        """ Return the list of existing dates
+
+        Date strings are picked from queued_on field corresponding to the
+        first task of each segment.
+        
+        Returns
+        -------
+        list of string
+        """
+        conn = sqlite3.connect(db_file,check_same_thread=True)
+        conn.text_factory=str
+        with conn:
+            l = conn.execute('select queued_on from tasks group by queued_on').fetchall()
+        return l[0]
+
+
+def add_tag (db_file, segid, tag):
+    """ Add new tag to the database
+    
+    Check that the tag do not exist yet for this id
+    
+    Parameters
+    ----------
+    segid: string. List of seg_id - separated. (- at the end of string also)
+    tag: string. Tags are ; separated in the db. 
+    """
+    seglst = segid.split('-')
+    conn = sqlite3.connect(db_file,check_same_thread=True)
+    conn.text_factory=str
+    for segid in seglst:
+        if segid:
+            with conn:
+                l = conn.execute('select tag,curr_dir from segments where seg_id=?',(segid,)).fetchone()
+                lst_tag = []
+                if l[0] is not None and l:
+                    lst_tag = l[0].split(";")
+                lst_tag.append(tag)
+                lst_tag = list(set(lst_tag))
+                str_tag = ";".join(lst_tag)
+                conn.execute('update segments set tag=? where seg_id=?',(str_tag,segid))
+            fn = glob(os.path.join(l[1], "*.meta"))
+            _update_meta(fn, str_tag)
+
+
+
+def _update_meta(fn, str_tag):
+        """ Update meta file with current tag value. 
+        
+        fn: string,  meta file name
+        tag: string, tag string
+        """
+        if fn:
+            with closing(file(fn[0], 'r')) as f:
+                d = pickle.load(f)
+            d['tag'] = str_tag
+            with closing(file(fn[0], 'w')) as f:    
+                r = pickle.dump(d,f)
+def del_tag (db_file, tag):
+        """ Delete tag from the database
+
+        Parameters
+        ----------
+        tag: string. 
+        """
+        conn = sqlite3.connect(db_file,check_same_thread=True)
+        conn.text_factory=str
+        with conn:
+            l = conn.execute('select seg_id, tag, curr_dir from segments where tag like ?',("%"+tag+"%",)).fetchall()
+            for s in l:
+                lst_tag = s[1].split(";")
+                lst_tag.remove(tag)
+                str_tag = ";".join(lst_tag)
+                conn.execute('update segments set tag=? where seg_id=?',(str_tag,s[0]))
+                fn = glob(os.path.join(s[2], "*.meta"))
+                _update_meta(fn, str_tag)
+
+        
+
+def _delseg(db_file, lst_seg):
+        """ Delete a pipeline instance. 
+
+        Delete all segments and products directories of a pipeline instance. 
+        
+        Parameters
+        ----------
+        lst_seg: integer list
+        """
+        ## remove everyboby
+        conn = sqlite3.connect(db_file,check_same_thread=True)
+        segid = lst_seg[0]
+        with conn:
+            currdir  = conn.execute('select curr_dir from segments where seg_id = ?',(int(segid),)).fetchone()[0]
+            l = conn.execute('select seg_id from segments where curr_dir like ?',(currdir+'%',)) 
+            for e in l:
+                lst_seg.remove(e[0])
+
+            ## delete from tasks_relations
+            l = conn.execute('delete from task_relations where child_id in (select task_id from tasks where seg_id in (select seg_id from segments where segments.curr_dir like ?))',(currdir+'%',))
+            ## delete from tasks
+            l = conn.execute('delete from tasks where seg_id in (select seg_id from segments where segments.curr_dir like ?)',(currdir+'%',))
+            ## delete from segments_relations
+            l = conn.execute('delete from segment_relations where child_id in (select seg_id from segments where segments.curr_dir like ?)',(currdir+'%',))
+            ## delete from segments
+            l = conn.execute('delete from segments where curr_dir like ?',(currdir+'%',))
+                
+        conn.close()
+        try:
+            shutil.rmtree(currdir)
+        except:
+            pass
+        if lst_seg:
+            _delseg(db_file, lst_seg)
+
+
+def _deltask(db_file, lst_task):
+        """ Delete a tasks instances.
+
+        Delete all products directories of a tasks instance. 
+        
+        Parameters
+        ----------
+        lst_tag: string, list of pipe id '-' separated
+        """
+        ## remove everyboby
+        conn = sqlite3.connect(db_file,check_same_thread=True)
+        taskid = lst_task[0]
+        with conn:
+            str_input  = conn.execute('select str_input from tasks where task_id = ?',(int(taskid),)).fetchone()[0]
+            lst_task.remove(taskid)
+
+            ## delete from tasks_relations
+            l = conn.execute('delete from task_relations where child_id = ? ',(int(taskid),))
+            ## delete from tasks
+            l = conn.execute('delete from tasks where task_id = ?',(int(taskid),))
+        conn.close()
+        try:
+            shutil.rmtree(str_input)
+        except:
+            pass
+        if lst_task:
+            _deltask(db_file, lst_task)
+
+def _get_fathers(db_file, segid):
+        """ Append upstream pipeline instance paths to the current path.
+        
+        Return a list which contains all upstream segment instances
+        id for a given segment instance. This is used to print a pipeline 
+        tree view with all dependencies. 
+
+        Parameters
+        ----------
+        segid: id of the leaf segment.
+        
+        Returns
+        -------
+        list of segid, for the upstream segment instances.
+        """
+        lstid = [int(segid)]
+        conn  = sqlite3.connect(db_file,check_same_thread=True)
+        with conn:
+            fids = conn.execute(
+                'select father_id from segment_relations where child_id = ?'
+                ,(segid,)).fetchall()
+        conn.close()
+        if fids:
+            for l in fids:
+                lstid += _get_fathers(db_file,l[0])
+        return lstid
+
+def _get_children(db_file, segid):
+        """ Return the list of pipeline instance paths which depend on seg
+        
+        Return a list which contains all downstream segment instances
+        id for a given segment instance. This is used to delete all dependencies. 
+
+        Parameters
+        ----------
+        segid: id of the segment.
+        
+        Returns
+        -------
+        list of segid, for the downstream segment instances.
+        """
+        lstid = [int(segid)]
+        conn  = sqlite3.connect(db_file,check_same_thread=True)
+        with conn:
+            fids = conn.execute(
+                'select child_id from segment_relations where father_id = ?'
+                ,(segid,)).fetchall()
+        conn.close()
+        if fids:
+            for l in fids:
+                lstid += _get_children(db_file, l[0])
+        return lstid
+
+
+def _get_children_task(db_file, taskid):
+        """ Return the list of tasks which depend on task.
+        
+        Return a list which contains all downstream task instances
+        id for a given task instance. This is used to delete all dependencies. 
+
+        Parameters
+        ----------
+        taskid: id of the task
+        
+        Returns
+        -------
+        list of taskid, for the downstream task instances.
+        """
+        lstid = [int(taskid)]
+        conn  = sqlite3.connect(db_file,check_same_thread=True)
+        with conn:
+            fids = conn.execute(
+                'select child_id from task_relations where father_id = ?'
+                ,(taskid,)).fetchall()
+        conn.close()
+        if fids:
+            for l in fids:
+                lstid += _get_children_task(db_file, l[0])
+        return lstid
+

pipelet/pbview.py

+## Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr>  <betoule@apc.univ-paris7.fr>
+## 
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, see http://www.gnu.org/licenses/gpl.html
+
+try:
+    import sqlite3
+except ImportError:
+    import pysqlite2 as sqlite3
+import cPickle as pickle
+import os
+import glob
+import cherrypy
+import auth
+from pipelet.web import TreeView
+from copy import copy
+
+class PBView(TreeView):
+    """ A pipeline Web interface dedicated to Polarbear experiment
+
+    It extends the minimalist tree view with a search object panel.
+    """
+    
+    def __init__ (self,db_file, name):
+        TreeView.__init__(self, db_file, name)
+        self.js_file += ["../static/pb.js"]
+        self.lst_ids = self.get_ids()
+        self.lst_mjd,self.lst_patch,self.lst_obs = self.get_mjd()
+    
+    def get_ids (self):
+        conn = sqlite3.connect(self.db_file,check_same_thread=True)
+        conn.text_factory=str
+        with conn:
+            l1 = conn.execute('select seg_id,seg,param from segments where seg="stimtau"').fetchall()
+            l2 = conn.execute('select seg_id,seg,param from segments where seg="gatherstim"').fetchall()
+        conn.close()
+        ll = []
+        for e in l1+l2:
+            if e[2] is not None:#len(e[2])>41:
+                ll.append (e[1]+'@'+e[2][-41:])
+            else:
+                ll.append(e[1]+'@'+str(e[0]))
+        self.selected_id = l1[0][0]
+        self.selected_seg= l1[0][1]
+        self.selected_idtxt= ll[0]
+        return ll
+
+    def get_mjd (self):
+        conn = sqlite3.connect(self.db_file,check_same_thread=True)
+        conn.text_factory=str
+        ll = []
+        with conn:
+            l = conn.execute('select seg_id from segments where seg like "unpack%"').fetchall()
+            for s in l:
+                ll += conn.execute('select input,str_input from tasks where seg_id=? and status="done"',(s[0],)).fetchall()
+        conn.close()
+        lst_mjd  = []
+        lst_patch= []
+        lst_dir  = []
+        for e in ll:
+            i = pickle.loads(e[0])
+            lst_mjd  += [k[0] for k in i.values()] 
+            lst_patch+= [k[1] for k in i.values()]
+            tmplst    = os.listdir(e[1])
+            lst_dir  += [f for f in tmplst 
+                         if os.path.isdir(os.path.join(e[1],f)) ]
+
+        lst_mjd  = list(set(lst_mjd))
+        lst_patch= list(set(lst_patch))
+        lst_dir  = list(set(lst_dir))
+        return (lst_mjd,lst_patch, lst_dir)
+
+    @cherrypy.expose
+    @auth.pipeauth.read_access()
+    def get_folder (self, mjd):
+        conn = sqlite3.connect(self.db_file,check_same_thread=True)
+        conn.text_factory=str
+        segid = self.selected_id
+        with conn:
+            #l = conn.execute('select seg_id from segments where seg="stimtau"').fetchall()
+            #segid = l[0][0]
+            sqlr = 'select input,str_input from tasks where seg_id=%s and status="done" and input like "XXX%sXXX"'%(segid,mjd)
+            sqlr = sqlr.replace("XXX", "%")
+            l = conn.execute(sqlr).fetchall()
+        conn.close()
+        ll = ["stim"+str(pickle.loads(e[0]).values()[0][0]) for e in l]
+        self.lst_obs = ll
+        self.selected_mjd = mjd
+        raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
+
+    @cherrypy.expose
+    @auth.pipeauth.read_access()
+    def set_id (self, idtxt):
+        conn = sqlite3.connect(self.db_file,check_same_thread=True)
+        conn.text_factory=str
+        segid = self.selected_id
+        searchid = idtxt.split("@")[-1]
+        searchseg = idtxt.split("@")[0]
+        with conn:
+            sqlr = 'select seg_id,seg from segments where param like "XXX%sXXX" and seg="%s"'%(searchid[2:-1],searchseg)
+            sqlr = sqlr.replace("XXX", "%")
+            l = conn.execute(sqlr).fetchone()
+        conn.close()
+        self.selected_idtxt = idtxt
+        if idtxt not in self.lst_ids:
+            self.lst_ids.append(idtxt)
+        self.selected_id = l[0]
+        self.selected_seg = l[1]
+        raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
+
+
+
+    def filter_panel (self, l):
+        """ Return an html string containing the filter panel
+        """
+        
+        html  = TreeView.filter_panel(self, l)
+        html += '<div id=right><fieldset id="right"><legend><span class="text">Search</span></legend>'
+        html += '<table width="100%">'
+
+        ## Object chkbox
+        html += '<tr  valign="top" id="tr_seg_id" class="filter"> <td style="width:150px;"><li><span class="smallcap">segment</span></li></td>'
+        ##<input id="cb_seg" type="checkbox"/><label></label></td>'
+
+        ## Object select
+        html += '<td style="width:50px;"><select id="se_seg_id" onchange=javascript:updateid();>'
+        for t in self.lst_ids:
+            if t==self.selected_idtxt:
+                html +='<option selected value="%s">%s</option>'%(self.selected_idtxt,self.selected_idtxt)
+            else:
+                html +='<option value="%s">%s</option>'%(t, t)
+        
+        html += '</select></td>'
+
+        ## Input chkbox
+        dd = dict({"mjd":(self.lst_mjd,"mjd"), "patch":(self.lst_patch,"patchname"), "folder":(self.lst_obs, "folder")})
+        for k,v in dd.iteritems():
+            
+            html += '<tr  valign="top" id="tr_id" class="filter"> <td style="width:150px;"><li><span class="smallcap">%s</span></li></td>'%(v[1])
+##<input id="cb_%s" type="checkbox"/><label>%s</label></td>'%(k,v[1])
+            if k=='mjd':
+                html += '<td style="width:50px;"><select id="se_%s_id" onchange=javascript:updatemjd();>'%k
+            else:
+                html += '<td style="width:50px;"><select id="se_%s_id">'%k
+            vlist = copy(v[0])
+            if hasattr(self, 'selected_mjd') and k=="mjd":
+                html +='<option selected value="%s">%s</option>'%(self.selected_mjd, self.selected_mjd)
+                
+                vlist.remove(self.selected_mjd)
+            for t in vlist:#v[0]:
+                html +='<option value="%s">%s</option>'%(t, t)
+            
+            html += '</select></td>'
+
+        ## Buttons
+        html += '<td class="buttons">'
+        html += '<a class="icon apply"  href="javascript:ssearch();"><small>Apply</small></a>'
+        html += '</td><tr></tr></table>'
+       
+        html += '</fieldset></div>'
+        return html
+
+    @cherrypy.expose
+    @auth.pipeauth.read_access()
+    def search(self , mjd=None, folder=None, patch=None,obj=None):
+        """
+
+        """
+        import cPickle as pickle
+        conn = sqlite3.connect(self.db_file,check_same_thread=True)
+        conn.text_factory=str
+
+        if self.selected_seg=='stimtau':#folder != "None" and mjd != "None":
+
+            foldernum = int(folder.replace("stim", ""))
+            with conn:
+                l = conn.execute('select seg_id,curr_dir from segments where seg_id=?',(self.selected_id,)).fetchall()
+                segid   = l[0][0]
+                currdir = l[0][1]
+                sqlr  = 'select str_input,task_id from tasks where seg_id="%s" and status="done" and input like "XXX%sXXX%sXXX"'%(segid,pickle.dumps(foldernum)[:-1], mjd)
+                sqlr  = sqlr.replace("XXX", "%")
+                l = conn.execute(sqlr).fetchall()
+            conn.close()
+            r = os.path.relpath(l[0][0], currdir)
+            directory = r.replace("../",'')
+            html = self.pipedir(segid=segid, directory=directory)
+
+
+        # elif folder =="None" and mjd != "None":
+        #     with conn:
+        #         l = conn.execute('select seg_id,curr_dir from segments where seg=?',("unpack_obs_stimulator",)).fetchall()
+        #         segid   = l[0][0]
+        #         currdir = l[0][1]
+        #         sqlr  = 'select str_input,task_id from tasks where seg_id="%s" and status="done" and input like "XXX%sXXX"'%(segid,mjd)
+        #         sqlr  = sqlr.replace("XXX", "%")
+        #         l = conn.execute(sqlr).fetchall()
+        #     conn.close()
+        #     r = os.path.relpath(l[0][0], currdir)
+        #     directory = r.replace("../",'')
+        #     html = self.pipedir(segid=segid, directory=directory)
+        
+        elif self.selected_seg=='gatherstim':
+            with conn:
+                l = conn.execute('select seg_id,curr_dir from segments where seg_id=?',(self.selected_id,)).fetchall()
+                segid   = l[0][0]
+                currdir = l[0][1]
+                sqlr  = 'select str_input from tasks where seg_id="%s" and status="done"'%(segid)
+                l = conn.execute(sqlr).fetchall()
+            conn.close()
+            r = os.path.relpath(l[0][0], currdir)
+            directory = r.replace("../",'')
+            html = self.pipedir(segid=segid, directory=directory)
+
+        return html

pipelet/static/mktree.css

@@ -87,6 +87,20 @@ a{
 }
 a:hover { text-decoration: underline; }
 
+#right{
+    float:left;
+    width:55%;
+}
+#left{
+    float:left;
+    width:45%;
+}
+
+
+#thumb{
+    width:95%;
+}
+
 #pipelist{
     width:60%;
 }
@@ -127,4 +141,10 @@ h1{
 /* tag display style */
 .pipetag{
     text-shadow:2px 2px #FF0000;
+}
+
+.smallcap{
+    font-weight: bold; 
+    font-family:Helvetica,Arial,sans-serif;
+    font-variant: small-caps
 }
\ No newline at end of file

pipelet/static/pb.js

+// Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr>  <betoule@apc.univ-paris7.fr>
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, see http://www.gnu.org/licenses/gpl.html
+
+// filter pipelines
+function ssearch(){
+    //var cb1 = document.getElementById('cb_mjd') ;
+    //var is1 = cb1.checked;
+    var mjd = "None";
+    //if (is1) 
+    //{
+    	var seltag = document.getElementById('se_mjd_id') ;
+    	mjd = seltag.options[seltag.selectedIndex].text; 
+    //}
+    //var cb2 = document.getElementById('cb_folder') ;
+    //var is2 = cb2.checked;
+    var folder = "None";
+    //if (is2) 
+    //{
+    	var seldate = document.getElementById('se_folder_id') ;
+    	folder = seldate.options[seldate.selectedIndex].text; 
+    //}
+    var url = "search?mjd="+mjd+"&folder="+folder;
+    //var url = "blala";
+    self.location.href=url;
+}
+
+function updatemjd () {
+    var seltag = document.getElementById('se_mjd_id') ;
+    var mjd = seltag.options[seltag.selectedIndex].text; 
+    self.location.href="get_folder?mjd="+mjd;
+    
+}
+function updateid () {
+    var seltag = document.getElementById('se_seg_id') ;
+    var seg = seltag.options[seltag.selectedIndex].text; 
+    self.location.href="set_id?idtxt="+seg;
+}

pipelet/static/tag.js

@@ -91,7 +91,6 @@ function del_prod(segid) {
   }
   self.location.href=url;
 }
-
 // filter pipelines
 function filter(){
   
@@ -114,3 +113,4 @@ function filter(){
   var url = "filter?tag="+tag+"&date="+date;
   self.location.href=url;
 }