Skip to content

GitLab

Commit e427abfa authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

Feature #868 closed. Fine tuning probably needed

parent 0c642e60
Hide whitespace changes
Inline Side-by-side
Showing with 191 additions and 30 deletions
pipelet/static/tag.js
View file @ e427abfa
...@@ -40,13 +40,13 @@ function del_seg (){ ...@@ -40,13 +40,13 @@ function del_seg (){
} }
// set new tag for checked segments // set new tag for checked segments
function edit_tag (){ function edit_tag (str_lst_tag){
var check = prompt("Select an existing tag among: \n - tag1\n - tag2\n or set a new tag for this pipe","tag1"); var check = prompt("Select an existing tag among:"+str_lst_tag+" or set a new tag for this pipe","tag1");
var url="addtag?segid="; var url="addtag?segid=";
lst = document.getElementsByName('checkbox'); lst = document.getElementsByName('checkbox');
for (var i=0;i<lst.length;i++){ for (var i=0;i<lst.length;i++){
if(lst[i].checked){ if(lst[i].checked){
url = url+lst[i].id+"--"; url = url+lst[i].id+";";
} }
} }
url = url+"&tag="+check; url = url+"&tag="+check;
...@@ -60,3 +60,36 @@ function uncheck (){ ...@@ -60,3 +60,36 @@ function uncheck (){
lst[i].checked=0; lst[i].checked=0;
} }
} }
// delete tag from sql base
function del_tag() {
var seltag = document.getElementById('se_tag_id') ;
var tag = seltag.options[seltag.selectedIndex].text;
var check = confirm("Delete tag "+tag+" ?");
if (check == true) {
self.location.href="deltag?tag="+tag;
}
}
// filter pipelines
function filter(){
var cbtag = document.getElementById('cb_tag') ;
var istag = cbtag.checked;
var tag = "None"
if (istag)
{
var seltag = document.getElementById('se_tag_id') ;
tag = seltag.options[seltag.selectedIndex].text;
}
var cbdate = document.getElementById('cb_date') ;
var isdate = cbdate.checked;
var date = "None"
if (isdate)
{
var seldate = document.getElementById('se_date_id') ;
date = seldate.options[seldate.selectedIndex].text;
}
var url = "filter?tag="+tag+"&date="+date;
self.location.href=url;
}
pipelet/tracker.py
View file @ e427abfa
...@@ -21,6 +21,7 @@ import shutil ...@@ -21,6 +21,7 @@ import shutil
import threading import threading
import Queue import Queue
import logging import logging
from utils import str_date
logger = logging.getLogger("scheduler") logger = logging.getLogger("scheduler")
class Tracker: class Tracker:
...@@ -306,13 +307,13 @@ class SqliteTracker(Tracker,threading.Thread): ...@@ -306,13 +307,13 @@ class SqliteTracker(Tracker,threading.Thread):
""" """
task_output = pickle.dumps(t.task_output) task_output = pickle.dumps(t.task_output)
task_input = pickle.dumps(t.task_input) task_input = pickle.dumps(t.task_input)
str_input = self.pipe.get_data_dir(t.seg, prod=t.task_input) str_input = self.pipe.get_data_dir(t.seg, prod=t.task_input)
with self.conn_lock: with self.conn_lock:
with self.conn: with self.conn:
seg_id = self.seg_id_cache[t.seg] seg_id = self.seg_id_cache[t.seg]
c = self.conn.execute('insert into tasks(seg_id, status, input, output, str_input) values(?, ?, ?, ?, ?)', c = self.conn.execute('insert into tasks(seg_id, status, input, output, str_input, queued_on) values(?, ?, ?, ?, ?, ?)',
(seg_id, 'queued', task_input, task_output, str_input)) (seg_id, 'queued', task_input, task_output, str_input, str_date()))
t.id = c.lastrowid t.id = c.lastrowid
logger.debug('for task:\n%s\nparents:\n%s\n'%(str(t),str(t.parents))) logger.debug('for task:\n%s\nparents:\n%s\n'%(str(t),str(t.parents)))
self.conn.executemany('insert into task_relations (father_id, child_id) values (?, ?)',((fid, t.id) for fid in t.parents)) self.conn.executemany('insert into task_relations (father_id, child_id) values (?, ?)',((fid, t.id) for fid in t.parents))
......
pipelet/utils.py
View file @ e427abfa
...@@ -23,7 +23,7 @@ import subprocess ...@@ -23,7 +23,7 @@ import subprocess
import tempfile import tempfile
import base64 import base64
import zlib import zlib
import datetime
def flatten(seq): def flatten(seq):
""" """
...@@ -103,6 +103,18 @@ def str_web(a): ...@@ -103,6 +103,18 @@ def str_web(a):
return s return s
def str_date():
""" Return a string representation of the date time now.
Returns:
--------
string %y/%m/%d %H:%M:%s
"""
d = datetime.datetime.now()
return (d.strftime("%Y-%m-%d %H:%M:%S"))
def is_code_file(s): def is_code_file(s):
"""Check wether the file is a python code file based an the file """Check wether the file is a python code file based an the file
extension. extension.
......
pipelet/web.py
View file @ e427abfa
...@@ -25,6 +25,7 @@ import shutil ...@@ -25,6 +25,7 @@ import shutil
from cherrypy.lib.static import serve_file from cherrypy.lib.static import serve_file
from auth import read_access, write_access from auth import read_access, write_access
import re import re
import pylab
html_tmp = """ html_tmp = """
<html> <html>
...@@ -56,30 +57,132 @@ class Web: ...@@ -56,30 +57,132 @@ class Web:
def get_lst_tag (self): def get_lst_tag (self):
""" Return the list of existing tags """ Return the list of existing tags
Tags are ; separated in the db.
Returns
-------
list of string
""" """
## sql blabla conn = sqlite3.connect(self.db_file,check_same_thread=True)
return [] conn.text_factory=str
lst = []
with conn:
l = conn.execute('select tag from segments').fetchall()
for s in l:
if s[0] is not None:
str_tag = s[0].split(";")
for e in str_tag:
if e:
lst.append(e)
return lst
def get_lst_date (self): def get_lst_date (self):
""" Return the list of existing dates """ Return the list of existing dates
Date strings are picked from queued_on field corresponding to the
first task of each segment.
Returns
-------
list of string
""" """
## sql blabla conn = sqlite3.connect(self.db_file,check_same_thread=True)
return [] conn.text_factory=str
lst = []
with conn:
l = conn.execute('select seg_id from segments').fetchall()
for s in l:
e = conn.execute('select queued_on from tasks where seg_id=?',(s[0],)).fetchone()
if e is not None and e:
lst.append(e[0])
return lst
@cherrypy.expose @cherrypy.expose
@write_access @write_access
def addtag (self, segid, tag): def addtag (self, segid, tag):
""" Add new tag to the database """ Add new tag to the database
Check that the tag do not exist yet for this id
Parameters
----------
segid: string. List of seg_id ; separated. (; at the end of string also)
tag: string. Tags are ; separated in the db.
""" """
## sql blabla seglst = segid.split(';')
seglst = segid.split('--') conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
for segid in seglst: for segid in seglst:
if segid: if segid:
## check that the tag do not exist yet for this id with conn:
pass l = conn.execute('select tag from segments where seg_id=?',(segid,)).fetchone()
lst_tag = []
if l[0] is not None and l:
lst_tag = l[0].split(";")
lst_tag.append(tag)
lst_tag = pylab.unique(lst_tag)
str_tag = ";".join(lst_tag)
conn.execute('update segments set tag=? where seg_id=?',(str_tag,segid))
raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
@cherrypy.expose
@write_access
def deltag (self, tag):
""" Delete tag from the database
Parameters
----------
tag: string.
"""
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
with conn:
l = conn.execute('select seg_id, tag from segments where tag like ?',("%"+tag+"%",)).fetchall()
for s in l:
lst_tag = s[1].split(";")
lst_tag.remove(tag)
str_tag = ";".join(lst_tag)
conn.execute('update segments set tag=? where seg_id=?',(str_tag,s[0]))
raise cherrypy.HTTPRedirect('/'+self.name+'/',303) raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
@cherrypy.expose
@read_access
def filter (self, tag=None, date=None):
""" Print the pipeline instances matching tag and date.
If both parameters are set, the selection
corresponds to the union.
Tag are set for leafs only. Upstream parents are automatically
added to the list.
Parameters
----------
tag: string
date: string
"""
lstseg = []
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
with conn:
if tag is not None:
l = conn.execute('select seg_id, tag from segments where tag like ?',("%"+tag+"%",)).fetchall()
for s in l:
lst_tag = s[1].split(";")
if tag in lst_tag:
lstseg+=self._get_highlight(s[0])
if date is not None:
l = conn.execute('select seg_id from tasks where queued_on = ?',(date,)).fetchall()
for s in l:
lstseg+=self._get_highlight(s[0])
html = self.index(highlight=lstseg)
return html
@cherrypy.expose @cherrypy.expose
@read_access @read_access
def index(self, highlight=None): def index(self, highlight=None):
...@@ -96,7 +199,7 @@ class Web: ...@@ -96,7 +199,7 @@ class Web:
conn.text_factory=str conn.text_factory=str
# get all instances # get all instances
with conn: with conn:
l = conn.execute('select seg, curr_dir, seg_id, tag from segments order by curr_dir').fetchall() l = conn.execute('select seg, curr_dir, seg_id, param from segments order by curr_dir').fetchall()
html = html_tmp html = html_tmp
...@@ -107,17 +210,21 @@ class Web: ...@@ -107,17 +210,21 @@ class Web:
html += '<table width="100%"><tr><td><table>' html += '<table width="100%"><tr><td><table>'
## Tag checkbox ## Tag checkbox
html += '<tr valign="top" id="tr_tag_id" class="filter"> <td style="width:150px;"><input id="cb_tag" type="checkbox"/><label for="cb_tag_id">Tag</label></td>' html += '<tr valign="top" id="tr_tag_id" class="filter"> <td style="width:100px;"><input id="cb_tag" type="checkbox"/><label>Tag</label></td>'
## Tag select ## Tag select
lst_tag = self.get_lst_tag() lst_tag = self.get_lst_tag()
html += '<td style="width:150px;"><select id="se_tag_id">' html += '<td style="width:50px;"><select id="se_tag_id">'
for t in lst_tag: for t in lst_tag:
html +='<option value="%s">%s</option>'%(t[0], t) html +='<option value="%s">%s</option>'%(t, t)
html += '</select></td>' html += '</select></td>'
## Tag button
html += '<td style="width:150px;"><a class="icon delete" href="javascript:del_tag();"><small>Delete</small></a></td>'
## Date checkbox ## Date checkbox
html += '<tr valign="top" id="tr_date_id" class="filter"> <td style="width:150px;"><input id="cb_date" type="checkbox"/><label for="cb_date_id">Date</label></td>' html += '<tr valign="top" id="tr_date_id" class="filter"> <td style="width:100px;"><input id="cb_date" type="checkbox"/><label>Date</label></td>'
## Date select ## Date select
lst_date = self.get_lst_date() lst_date = self.get_lst_date()
...@@ -126,14 +233,19 @@ class Web: ...@@ -126,14 +233,19 @@ class Web:
html +='<option value="%s">%s</option>'%(t[0], t) html +='<option value="%s">%s</option>'%(t[0], t)
html += '</select></td>' html += '</select></td>'
str_lst_tag = "\\n"
for t in lst_tag:
str_lst_tag += "- "+t+"\\n"
## Buttons ## Buttons
html += '<tr></tr>' html += '<tr></tr>'
html += '</table>' html += '</table>'
html += '</td><td></td></tr></table>' html += '</td><td></td></tr></table>'
html += '<p class="buttons">' html += '<p class="buttons">'
html += '<a class="icon apply" href="#"><small>Apply</small></a>' html += '<a class="icon apply" href="javascript:filter();"><small>Apply</small></a>'
html += '<a class="icon clear" href="javascript:uncheck();"><small>Clear</small></a>' html += '<a class="icon clear" href="javascript:uncheck();"><small>Clear</small></a>'
html += '<a class="icon tag" href="javascript:edit_tag();"><small>Tag</small></a>' html += '<a class="icon tag" href="javascript:edit_tag(\'%s\');"><small>Tag</small></a>'%str_lst_tag
html += '<a class="icon delete" href="javascript:del_seg();"><small>Delete</small></a>' html += '<a class="icon delete" href="javascript:del_seg();"><small>Delete</small></a>'
html += '<a class="icon log" href="log?logdir=%s"><small>Browse log</small></a>'%(l[0][1].split("seg")[0]+"log") html += '<a class="icon log" href="log?logdir=%s"><small>Browse log</small></a>'%(l[0][1].split("seg")[0]+"log")
html +='</p></fieldset>' html +='</p></fieldset>'
...@@ -164,7 +276,6 @@ class Web: ...@@ -164,7 +276,6 @@ class Web:
for stat in e: for stat in e:
ss = '<a href="product?segid=%s&status=%s" class=%s>%d</a>, '%(s[2],stat[0], stat[0], stat[1]) + ss ss = '<a href="product?segid=%s&status=%s" class=%s>%d</a>, '%(s[2],stat[0], stat[0], stat[1]) + ss
ss += '<INPUT type="checkbox" name="checkbox" id="%d"'%(s[2]) ss += '<INPUT type="checkbox" name="checkbox" id="%d"'%(s[2])
#ss += '<a href="delseg?segid=%d"> (delete)</a>'%(s[2])
diff = s[1].count('/') - indent diff = s[1].count('/') - indent
if diff == 1: if diff == 1:
html += '<ul> <li id=%d ><a href="code?segid=%d">%s</a> : %s\n'%(s[2],s[2],s[0],ss) html += '<ul> <li id=%d ><a href="code?segid=%d">%s</a> : %s\n'%(s[2],s[2],s[0],ss)
...@@ -212,9 +323,10 @@ class Web: ...@@ -212,9 +323,10 @@ class Web:
def _get_highlight(self, segid): def _get_highlight(self, segid):
""" Append upstream pipeline instance paths to the current path. """ Append upstream pipeline instance paths to the current path.
Return a string which contains all upstream segment instances Return a list which contains all upstream segment instances
id for a given segment instance. This is used to print a pipeline id for a given segment instance. This is used to print a pipeline
tree view with all dependencies. tree view with all dependencies.
Parameters Parameters
---------- ----------
segid: id of the leaf segment. segid: id of the leaf segment.
...@@ -223,14 +335,17 @@ class Web: ...@@ -223,14 +335,17 @@ class Web:
------- -------
list of segid, for the upstream segment instances. list of segid, for the upstream segment instances.
""" """
## TO DO : get subpipes somewhere ??? lstid = [segid]
conn = sqlite3.connect(self.db_file,check_same_thread=True) conn = sqlite3.connect(self.db_file,check_same_thread=True)
with conn: with conn:
fids = conn.execute( fids = conn.execute(
'select father_id from segment_relations where child_id = ?' 'select father_id from segment_relations where child_id = ?'
,(segid,)).fetchall() ,(segid,)).fetchall()
conn.close() conn.close()
return [int(segid)]+[i for i in fids] if fids:
for l in fids:
lstid += self._get_highlight(l[0])
return lstid
@cherrypy.expose @cherrypy.expose
@read_access @read_access
...@@ -272,11 +387,11 @@ class Web: ...@@ -272,11 +387,11 @@ class Web:
Parameters Parameters
---------- ----------
segid: string, list of pipe id '--' separated segid: string, list of pipe id ';' separated
""" """
cherrypy.log.error_log.warning('called ONCE') cherrypy.log.error_log.warning('called ONCE')
if segid is not None: if segid is not None:
seglist = segid.split("--") seglist = segid.split(";")
for segid in seglist: for segid in seglist:
if segid: if segid:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment