Skip to content

GitLab

Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 3676ae1e authored by Betoule Marc's avatar Betoule Marc
Browse files

huge performance improvements in index, requires the creation of 2 index on table tasks: 

create index task_idx2 on tasks(queued_on);
create index task_idx1 on tasks(seg_id, status);
parent a331e481
Hide whitespace changes
Inline Side-by-side
Showing with 25 additions and 21 deletions
pipelet/tracker.py
View file @ 3676ae1e
......@@ -375,12 +375,12 @@ class SqliteTracker(Tracker,threading.Thread):
# May hurt performances very badly for short tasks
self._asynchronous_request('update segments set param=? where seg_id =?',
(t.param, self.seg_id_cache[t.seg]))
#fn = self.pipe.get_meta_file(t.seg)
#with closing(file(fn, 'r')) as f:
# d = pickle.load(f)
#d['param'] = t.param
#with closing(file(fn, 'w')) as f:
# r = pickle.dump(d,f)
fn = self.pipe.get_meta_file(t.seg)
with closing(file(fn, 'r')) as f:
d = pickle.load(f)
d['param'] = t.param
with closing(file(fn, 'w')) as f:
r = pickle.dump(d,f)
if status =='done' or status == 'failed':
t.store_meta(self.pipe.get_meta_file(t.seg, prod=t.task_input))
return t
......
pipelet/web.py
View file @ 3676ae1e
......@@ -27,7 +27,7 @@ from cherrypy.lib.static import serve_file
import auth
import re
import pickle
from cherrypy.lib import profiler
html_tmp = """
<html>
<head>
......@@ -44,7 +44,7 @@ class Web:
The informations about the pipeline are retrieve from its data base file.
Pipelines are printed through a tree view with links to all segments and products.
"""
p = profiler.Profiler('/home/betoule/pipewebprof/')
## boolean, Web object is exposed.
exposed = True
def __init__(self, db_file, name):
......@@ -94,14 +94,9 @@ class Web:
"""
conn = sqlite3.connect(self.db_file,check_same_thread=True)
conn.text_factory=str
lst = []
with conn:
l = conn.execute('select seg_id from segments').fetchall()
for s in l:
e = conn.execute('select queued_on from tasks where seg_id=?',(s[0],)).fetchone()
if e is not None and e:
lst.append(e[0])
return list(set(lst))
l = conn.execute('select queued_on from tasks group by queued_on').fetchall()
return l
@cherrypy.expose
@auth.pipeauth.write_access()
......@@ -231,7 +226,10 @@ class Web:
@cherrypy.expose
@auth.pipeauth.read_access()
def index(self, highlight=None, thumbnail=None):
def index(self, **keys):
return self.p.run(self._index,**keys)
def _index(self, highlight=None, thumbnail=None):
""" Pipeline instances tree view
Print the pipeline instances trough a tree view.
......@@ -245,6 +243,10 @@ class Web:
# get all instances
with conn:
l = conn.execute('select seg, curr_dir, seg_id, param from segments order by curr_dir').fetchall()
tcounts = conn.execute('select seg_id, status, count() from tasks group by seg_id, status order by seg_id').fetchall()
sindex = [e[0] for e in tcounts]
index = [[] for i in range(max(sindex)+1)]
for i, s in enumerate(sindex): index[s].append(i)
html = html_tmp
html += '<a class="icon home" href="/%s/"><small>Home</small></a>'%(self.name)
......@@ -327,14 +329,15 @@ class Web:
cherrypy.log.error_log.warning(str(s[2])+str(highlight))
l = newl
for s in l:
with conn:
e = conn.execute('select status, count(status) from tasks where seg_id=? group by status',(s[2],)).fetchall()
ss = s[3]
#with conn:
# e = conn.execute('select status, count(status) from tasks where seg_id=? group by status',(s[2],)).fetchall()
e = [tcounts[i] for i in index[s[2]]]
ss = s[3]
if ss is None:
ss = ""
ss = '<a class="icon image" href="index?thumbnail=%d"></a>'%(s[2])+ss
for stat in e:
ss = '<a href="product?segid=%s&status=%s" class=%s>%d</a>, '%(s[2],stat[0], stat[0], stat[1]) + ss
ss = '<a href="product?segid=%s&status=%s" class=%s>%d</a>, '%(s[2],stat[1], stat[1], stat[2]) + ss
ss += '<INPUT type="checkbox" name="checkbox" id="%d"'%(s[2])
diff = s[1].count('/') - indent
......@@ -903,7 +906,8 @@ def main():
'log.access_file':options.access_file,
'server.pidfile':options.pid_file,
'pipeauth.read_access.auth_type':options.authentication_type,
'pipeauth.write_access.auth_type':options.authentication_type}
'pipeauth.write_access.auth_type':options.authentication_type,
}
}
if options.no_daemon:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment