web.py 24.6 KB
Newer Older
Maude Le Jeune's avatar
Maude Le Jeune committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
## Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr>  <betoule@apc.univ-paris7.fr>
## 
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, see http://www.gnu.org/licenses/gpl.html


17 18 19 20
import sqlite3
import cherrypy
from contextlib import closing
import os.path
21
import os
22 23
current_dir = os.path.dirname(os.path.abspath(__file__))
from glob import glob
24
import shutil
25
from cherrypy.lib.static import serve_file
26
from auth import read_access, write_access
Marc Betoule's avatar
Marc Betoule committed
27
import re
28
import pylab
29 30 31 32

html_tmp = """
<html>
<head>
Betoule Marc's avatar
Betoule Marc committed
33 34
<SCRIPT SRC="/static/mktree.js" LANGUAGE="JavaScript"></SCRIPT>
<SCRIPT SRC="/static/cook.js" LANGUAGE="JavaScript"></SCRIPT>
Maude Le Jeune's avatar
Maude Le Jeune committed
35
<SCRIPT SRC="/static/tag.js" LANGUAGE="JavaScript"></SCRIPT>
Betoule Marc's avatar
Betoule Marc committed
36
<LINK REL="stylesheet" HREF="/static/mktree.css">  
37 38
</head><body>
"""
39
class Web:
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
40 41 42 43 44 45
    """ A pipeline Web interface.
    
    A pipeline Web interface allows to browse all instances of a given pipeline. 
    The informations about the pipeline are retrieve from its data base file. 
    Pipelines are printed through a tree view with links to all segments and products. 
    """
Betoule Marc's avatar
Betoule Marc committed
46
    exposed = True
Betoule Marc's avatar
Betoule Marc committed
47
    def __init__(self, db_file, name):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
48 49 50 51 52 53
        """ Initialize a web instance. 
        
        Parameters
        ----------
        db_file: string, file name of the pipe data base.
        """
Betoule Marc's avatar
Betoule Marc committed
54
        self.name = name
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
55
        ## Pipeline data base file.
56
        self.db_file = db_file
Maude Le Jeune's avatar
Maude Le Jeune committed
57
        
Maude Le Jeune's avatar
Maude Le Jeune committed
58
    def get_lst_tag (self):
Maude Le Jeune's avatar
Maude Le Jeune committed
59
        """ Return the list of existing tags
60 61 62 63 64 65

        Tags are ; separated in the db. 

        Returns
        -------
        list of string
Maude Le Jeune's avatar
Maude Le Jeune committed
66
        """
67 68 69 70 71 72 73 74 75 76 77 78 79
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        conn.text_factory=str
        lst = []
        with conn:
            l = conn.execute('select tag from segments').fetchall()
            for s in l:
                if s[0] is not None:
                    str_tag = s[0].split(";")
                    for e in str_tag:
                        if e:
                            lst.append(e)
        return lst
        
Maude Le Jeune's avatar
Maude Le Jeune committed
80 81

    def get_lst_date (self):
Maude Le Jeune's avatar
Maude Le Jeune committed
82
        """ Return the list of existing dates
83 84 85 86 87 88 89

        Date strings are picked from queued_on field corresponding to the
        first task of each segment.
        
        Returns
        -------
        list of string
Maude Le Jeune's avatar
Maude Le Jeune committed
90
        """
91 92 93 94 95 96 97 98 99 100
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        conn.text_factory=str
        lst = []
        with conn:
            l = conn.execute('select seg_id from segments').fetchall()
            for s in l:
                e = conn.execute('select queued_on from tasks where seg_id=?',(s[0],)).fetchone()
                if e is not None and e:
                    lst.append(e[0])
        return lst
Maude Le Jeune's avatar
Maude Le Jeune committed
101 102 103 104 105

    @cherrypy.expose
    @write_access
    def addtag (self, segid, tag):
        """ Add new tag to the database
106 107 108 109 110 111 112
        
        Check that the tag do not exist yet for this id
        
        Parameters
        ----------
        segid: string. List of seg_id ; separated. (; at the end of string also)
        tag: string. Tags are ; separated in the db. 
Maude Le Jeune's avatar
Maude Le Jeune committed
113
        """
114 115 116
        seglst = segid.split(';')
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        conn.text_factory=str
Maude Le Jeune's avatar
Maude Le Jeune committed
117 118
        for segid in seglst:
            if segid:
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
                with conn:
                    l = conn.execute('select tag from segments where seg_id=?',(segid,)).fetchone()
                    lst_tag = []
                    if l[0] is not None and l:
                        lst_tag = l[0].split(";")
                    lst_tag.append(tag)
                    lst_tag = pylab.unique(lst_tag)
                    str_tag = ";".join(lst_tag)
                    conn.execute('update segments set tag=? where seg_id=?',(str_tag,segid))

        raise cherrypy.HTTPRedirect('/'+self.name+'/',303)

    @cherrypy.expose
    @write_access
    def deltag (self, tag):
        """ Delete tag from the database

        Parameters
        ----------
        tag: string. 
        """
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        conn.text_factory=str
        with conn:
            l = conn.execute('select seg_id, tag from segments where tag like ?',("%"+tag+"%",)).fetchall()
            for s in l:
                lst_tag = s[1].split(";")
                lst_tag.remove(tag)
                str_tag = ";".join(lst_tag)
                conn.execute('update segments set tag=? where seg_id=?',(str_tag,s[0]))

Maude Le Jeune's avatar
Maude Le Jeune committed
150
        raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
Maude Le Jeune's avatar
Maude Le Jeune committed
151
        
Maude Le Jeune's avatar
Maude Le Jeune committed
152

153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
    @cherrypy.expose
    @read_access
    def filter (self, tag=None, date=None):
        """ Print the pipeline instances matching tag and date.
        
        If both parameters are set, the selection
        corresponds to the union.
        Tag are set for leafs only. Upstream parents are automatically
        added to the list. 

        Parameters
        ----------
        tag: string
        date: string
        """
        lstseg = []
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        conn.text_factory=str
        with conn:
            if tag is not None:
                l = conn.execute('select seg_id, tag from segments where tag like ?',("%"+tag+"%",)).fetchall()
                for s in l: 
                    lst_tag = s[1].split(";")
                    if tag in lst_tag:
                        lstseg+=self._get_highlight(s[0])

            if date is not None:
                l = conn.execute('select seg_id from tasks where queued_on = ?',(date,)).fetchall()
                for s in l: 
                    lstseg+=self._get_highlight(s[0])
        html = self.index(highlight=lstseg)
        return html

186
    @cherrypy.expose
187
    @read_access
188
    def index(self, highlight=None):
Maude Le Jeune's avatar
Maude Le Jeune committed
189
        """ Pipeline instances tree view
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
190 191 192 193 194

        Print the pipeline instances trough a tree view.
        
        Parameters
        ---------- 
Marc Betoule's avatar
Marc Betoule committed
195
        highlight: list of segid (optional), filter the printed seg by segid.
Maude Le Jeune's avatar
Maude Le Jeune committed
196
        """
197

198
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
Marc Betoule's avatar
Marc Betoule committed
199
        conn.text_factory=str
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
200
        # get all instances
201
        with conn:
202
            l = conn.execute('select seg, curr_dir, seg_id, param from segments order by curr_dir').fetchall()
203 204
        
        html = html_tmp 
Maude Le Jeune's avatar
Maude Le Jeune committed
205 206 207 208 209 210 211 212
        
        html += '<h1>Pipelines in %s </h1>'%self.name

        ## Filter fieldset
        html += '<fieldset id="filters"><legend><span class="text">Filters</span></legend>'
        html += '<table width="100%"><tr><td><table>'
        
        ## Tag checkbox
213
        html += '<tr  valign="top" id="tr_tag_id" class="filter"> <td style="width:100px;"><input id="cb_tag" type="checkbox"/><label>Tag</label></td>'
Maude Le Jeune's avatar
Maude Le Jeune committed
214 215 216

        ## Tag select
        lst_tag = self.get_lst_tag()
217
        html += '<td style="width:50px;"><select id="se_tag_id">'
Maude Le Jeune's avatar
Maude Le Jeune committed
218
        for t in lst_tag:
219
            html +='<option value="%s">%s</option>'%(t, t)
Maude Le Jeune's avatar
Maude Le Jeune committed
220 221
        html += '</select></td>'

222 223 224 225
        ## Tag button
        html += '<td style="width:150px;"><a class="icon delete" href="javascript:del_tag();"><small>Delete</small></a></td>'
        

Maude Le Jeune's avatar
Maude Le Jeune committed
226
        ## Date checkbox
227
        html += '<tr  valign="top" id="tr_date_id" class="filter"> <td style="width:100px;"><input id="cb_date" type="checkbox"/><label>Date</label></td>'
Maude Le Jeune's avatar
Maude Le Jeune committed
228

Maude Le Jeune's avatar
Maude Le Jeune committed
229
        ## Date select
Maude Le Jeune's avatar
Maude Le Jeune committed
230 231 232 233 234 235
        lst_date = self.get_lst_date()
        html += '<td style="width:150px;"><select id="se_date_id">'
        for t in lst_date:
            html +='<option value="%s">%s</option>'%(t[0], t)
        html += '</select></td>'
        
236 237 238 239 240
        str_lst_tag = "\\n"
        for t in lst_tag:
            str_lst_tag += "- "+t+"\\n"


Maude Le Jeune's avatar
Maude Le Jeune committed
241
        ## Buttons
Maude Le Jeune's avatar
Maude Le Jeune committed
242 243 244 245
        html += '<tr></tr>'
        html += '</table>'
        html += '</td><td></td></tr></table>'
        html += '<p class="buttons">'
246
        html += '<a class="icon apply"  href="javascript:filter();"><small>Apply</small></a>'
Maude Le Jeune's avatar
Maude Le Jeune committed
247
        html += '<a class="icon clear"  href="javascript:uncheck();"><small>Clear</small></a>'
248
        html += '<a class="icon tag"    href="javascript:edit_tag(\'%s\');"><small>Tag</small></a>'%str_lst_tag
Maude Le Jeune's avatar
Maude Le Jeune committed
249
        html += '<a class="icon delete" href="javascript:del_seg();"><small>Delete</small></a>'
Maude Le Jeune's avatar
Maude Le Jeune committed
250 251 252
        html += '<a class="icon log" href="log?logdir=%s"><small>Browse log</small></a>'%(l[0][1].split("seg")[0]+"log")
        html +='</p></fieldset>'
        html += '<br><div class="list"><ul class="mktree" id="segtree">'
253
        
254
        indent = -1
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
255
        # select a subset
256 257 258
        if highlight is not None:
            newl = []
            for s in l:
Marc Betoule's avatar
Marc Betoule committed
259
                if s[2] in highlight:
260
                    newl.append(s)
Marc Betoule's avatar
Marc Betoule committed
261 262
                else:
                    cherrypy.log.error_log.warning(str(s[2])+str(highlight))
263
            l = newl
264
        for s in l:
265
            with conn:
Marc Betoule's avatar
Marc Betoule committed
266 267
                e = conn.execute('select status, count(status) from tasks where seg_id=? group by status',(s[2],)).fetchall()
                ss = s[3]
Marc Betoule's avatar
Marc Betoule committed
268 269 270 271 272
            #try:
            #    with closing(file(os.path.join(s[1],"stdout"))) as f:
            #        ss = f.read()
            #except IOError:
            #    ss = ""
273
            if ss is None:
Marc Betoule's avatar
Marc Betoule committed
274 275
                ss = ""
            print s
276
            for stat in e:
Marc Betoule's avatar
Marc Betoule committed
277
                ss = '<a href="product?segid=%s&status=%s" class=%s>%d</a>, '%(s[2],stat[0], stat[0], stat[1]) + ss
Maude Le Jeune's avatar
Maude Le Jeune committed
278
            ss += '<INPUT type="checkbox" name="checkbox" id="%d"'%(s[2])
279 280
            diff = s[1].count('/') - indent
            if diff == 1:
Marc Betoule's avatar
Marc Betoule committed
281
                html += '<ul> <li id=%d ><a href="code?segid=%d">%s</a> : %s\n'%(s[2],s[2],s[0],ss)
282
            elif diff == 0:
Marc Betoule's avatar
Marc Betoule committed
283
                html += '</li><li id=%d ><a href="code?segid=%d">%s</a> : %s\n'%(s[2],s[3],s[0],ss)
284
            elif diff > 1:
Marc Betoule's avatar
Marc Betoule committed
285
                html += '<li id=%d ><a href="code?segid=%d">%s</a> : %s\n'%(s[2],s[2],s[0],ss)
286
            else:
Marc Betoule's avatar
Marc Betoule committed
287
                html += '</li> </ul>\n'*abs(diff) + '<li id=%d> <a href="code?segid=%d">%s</a> : %s'%(s[2],s[2],s[0],ss)
288
            indent += diff 
289
        conn.close()
290
        html += '</li></ul>'*(indent-l[0][1].count('/')+1)
Maude Le Jeune's avatar
Maude Le Jeune committed
291 292
        
        
Marc Betoule's avatar
Marc Betoule committed
293 294
        if not highlight:
            html += '</div></body></html>'
295 296 297
        return html
    
    @cherrypy.expose
298
    @read_access
Marc Betoule's avatar
Marc Betoule committed
299
    def product(self, segid=None, status=None):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
300 301 302 303 304 305 306 307
        """ Products index.
        
        Print the content of the product directory.
        
        Parameters
        ----------
        currdir: string, pipeline instance path.
        status: string, product status ('done', 'queued', 'running', 'failed')
Maude Le Jeune's avatar
Maude Le Jeune committed
308
        """
309 310
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        with conn:
Marc Betoule's avatar
Marc Betoule committed
311 312 313
            seg, currdir = conn.execute(
                'select seg, curr_dir from segments where seg_id = ?'
                ,(segid,)).fetchone()
314
            l = conn.execute('select str_input from tasks where seg_id=? and status=?',(segid, status)).fetchall()
315
        conn.close()
Marc Betoule's avatar
Marc Betoule committed
316
        html = html_tmp + '<h1> Data products for %s tasks in segment %s </h1>' % (status, seg)
Betoule Marc's avatar
Betoule Marc committed
317
        html += '<div class="list"><p>Directory : %s</p>  %d <span class="%s">%s</span> tasks <ul> '%( currdir, len(l), status, status)
318
        for e in l:
Marc Betoule's avatar
Marc Betoule committed
319
            html += '<li><a href="pipedir?segid=%s&directory=%s"> %s </a></li>'%(segid,os.path.relpath(e[0],start=currdir),e[0])
Betoule Marc's avatar
Betoule Marc committed
320
        html += '</ul></div></body></html>'
321 322
        return html

Marc Betoule's avatar
Marc Betoule committed
323
    def _get_highlight(self, segid):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
324 325
        """ Append upstream pipeline instance paths to the current path.
        
326
        Return a list which contains all upstream segment instances
Marc Betoule's avatar
Marc Betoule committed
327
        id for a given segment instance. This is used to print a pipeline 
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
328
        tree view with all dependencies. 
329

Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
330 331
        Parameters
        ----------
Marc Betoule's avatar
Marc Betoule committed
332
        segid: id of the leaf segment.
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
333 334 335
        
        Returns
        -------
Marc Betoule's avatar
Marc Betoule committed
336
        list of segid, for the upstream segment instances.
337
        """
338 339
        lstid = [segid]
        conn  = sqlite3.connect(self.db_file,check_same_thread=True)
Betoule Marc's avatar
Betoule Marc committed
340
        with conn:
341 342
            fids = conn.execute(
                'select father_id from segment_relations where child_id = ?'
Betoule Marc's avatar
Betoule Marc committed
343 344
                ,(segid,)).fetchall()
        conn.close()
345 346 347 348
        if fids:
            for l in fids:
                lstid += self._get_highlight(l[0])
        return lstid
349

350
    @cherrypy.expose
351
    @read_access
Marc Betoule's avatar
Marc Betoule committed
352
    def code(self, segid=None):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
353 354 355 356 357 358 359
        """ Segment's files index. 
        
        Print the content of a segment directory (code files, parameters file, ...)
        
        Parameters
        ----------
        currdir: string, pipeline instance path.
Maude Le Jeune's avatar
Maude Le Jeune committed
360
        """
Marc Betoule's avatar
Marc Betoule committed
361 362 363 364 365 366 367 368 369
        highlight = self._get_highlight(segid)
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        with conn:
            seg, currdir = conn.execute(
                'select seg, curr_dir from segments where seg_id = ?'
                ,(segid,)).fetchone()
        conn.close()
        html = self.index(highlight=highlight) + '<p>%s</p><ul>' % (currdir)
        l = glob(os.path.join(currdir,"*.*"))
370
        for e in l:
Marc Betoule's avatar
Marc Betoule committed
371 372
            html += '<li><a href="download?segid=%d&filepath=%s"> %s </a></li>'%((int(segid),)+(os.path.basename(e),)*2)
        html += '</ul></div></body></html>'
373 374
        return html

Maude Le Jeune's avatar
Maude Le Jeune committed
375 376 377 378 379 380
    def del_from_checkbox ():
        """
        """
        


381
    @cherrypy.expose
382
    @write_access
Marc Betoule's avatar
Marc Betoule committed
383
    def delseg(self, segid=None):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
384 385 386 387 388 389
        """ Delete a pipeline instance. 

        Delete all segments and products directories of a given pipeline instance.
        
        Parameters
        ----------
390
        segid: string, list of pipe id ';' separated
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
391
        """
392
        cherrypy.log.error_log.warning('called ONCE')
Marc Betoule's avatar
Marc Betoule committed
393
        if segid is not None:
394
            seglist = segid.split(";")
Maude Le Jeune's avatar
Maude Le Jeune committed
395 396 397 398 399 400 401 402 403 404 405 406
       
            for segid in seglist:
                if segid:
                    conn = sqlite3.connect(self.db_file,check_same_thread=True)
                    with conn:
                        currdir  = conn.execute('select curr_dir from segments where seg_id = ?',(int(segid),)).fetchone()[0]
                        l = conn.execute('delete from tasks where seg_id in (select seg_id from segments where segments.curr_dir like ?)',(currdir+'%',))
                    conn.close()
                    try:
                        shutil.rmtree(currdir)
                    except:
                        pass
407
        
408
        raise cherrypy.HTTPRedirect('/'+self.name+'/',303)
409

410
    @cherrypy.expose
411
    @read_access
Marc Betoule's avatar
Marc Betoule committed
412
    def pipedir(self, segid=None, directory=None):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
413 414 415 416 417 418
        """ Print the content of a directory. 

        Parameters
        ----------
        directory: string pipeline directory path.
        """
Marc Betoule's avatar
Marc Betoule committed
419
        directory = self.check_path (segid, directory)
Betoule Marc's avatar
Betoule Marc committed
420
        html = html_tmp + '<h1> Content of %s </h1> <div class="list"><ul>'%directory
421 422
        for filename in glob(os.path.join(directory,'*')):
            absPath = os.path.abspath(filename)
Maude Le Jeune's avatar
Maude Le Jeune committed
423
            if os.path.islink(absPath):
424
                html += '<li><a href="pipedir?segid=%d&directory='%int(segid) + absPath + '">' + os.path.basename(filename)+"("+os.path.realpath(filename)+")" + "</a></li>"
Maude Le Jeune's avatar
Maude Le Jeune committed
425
            elif os.path.isdir(absPath):
Marc Betoule's avatar
Marc Betoule committed
426
                html += '<li><a href="pipedir?segid=%d&directory='%int(segid) + absPath + '">' + os.path.basename(filename) + "</a></li>"
427
            else:
Marc Betoule's avatar
Marc Betoule committed
428
                html += '<li><a href="download?segid=%d&filepath='%int(segid) + absPath + '">' + os.path.basename(filename) + "</a> </li>"
429
                
Betoule Marc's avatar
Betoule Marc committed
430
        html += """</ul></div></body></html>"""
431
        return html
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
432

Marc Betoule's avatar
Marc Betoule committed
433
    forbidden_path = re.compile('.*\.\..*')
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
434

435 436 437 438 439 440 441

    @cherrypy.expose
    @read_access
    def log(self, logdir):
        """ Print the content of the log directory. 
        """
        directory = logdir
Maude Le Jeune's avatar
Maude Le Jeune committed
442
        html = html_tmp + '<h1> Content of %s </h1> <a href="/%s/">Back</a><div class="list"><a class="icon delete" href="delete_log?logdir=%s"><small>Delete logs</small></a><ul>'%(directory,self.name,logdir)
443
        for filename in sorted(glob(os.path.join(directory,'*')), reverse=True):
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
            absPath = os.path.abspath(filename)
            html += '<li><a href="serve_log?filename='+absPath+ '">' + os.path.basename(filename) + "</a> </li>"
        html += """</ul></div></body></html>"""
        return html

    @cherrypy.expose
    @read_access
    def serve_log(self, filename):
        """ Print the content of the log file.
        """
        return serve_file(filename, content_type='text/plain', disposition="inline")

    @cherrypy.expose
    @write_access
    def delete_log(self, logdir):
        """ Delete the content of the log directory.
        """
        for filename in glob(os.path.join(logdir,'*')):
            absPath = os.path.abspath(filename)
            os.remove(absPath)
        raise cherrypy.HTTPRedirect("log?logdir=%s"%logdir,303)

Marc Betoule's avatar
Marc Betoule committed
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
    def check_path(self, segid, path):
        """Chroot the path to the segid currdir.
        """
        conn = sqlite3.connect(self.db_file,check_same_thread=True)
        with conn:
            seg, currdir = conn.execute(
                'select seg, curr_dir from segments where seg_id = ?'
                ,(segid,)).fetchone()
        conn.close()
        filepath = os.path.realpath(os.path.join(currdir, path))
        if self.forbidden_path.match(os.path.relpath(filepath, start=currdir)) is not None:
            raise cherrypy.HTTPError(403)
        else:
            return filepath
        
481
    @cherrypy.expose
482
    @read_access
Marc Betoule's avatar
Marc Betoule committed
483
    def download(self, segid=None, filepath=None):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
484 485 486 487 488 489 490 491
        """ Download a file.
        
        Text files are printed directly.
        
        Parameters
        ----------
        filepath: string, file name. 
        """
Marc Betoule's avatar
Marc Betoule committed
492
        filepath = self.check_path(segid, filepath)
493
        if os.path.splitext(filepath)[1] in ['.log','.txt','.list','.py']:
494
            return serve_file(filepath, content_type='text/plain', disposition="inline")
Marc Betoule's avatar
Marc Betoule committed
495
        else:
496
            return serve_file(filepath, disposition="inline")
497

498
class PipeIndex():
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
499 500 501 502 503
    """ A pipeline index Web interface.
    
    A pipeline index Web interface prints the lists of available pipelines 
    found in a given path. 
    """
Betoule Marc's avatar
Betoule Marc committed
504
    def __init__(self):
Betoule Marc's avatar
Betoule Marc committed
505
        self.pipelist = []
Betoule Marc's avatar
Betoule Marc committed
506
        for pipe, dbfile in cherrypy.config['Pipelines'].iteritems():
Betoule Marc's avatar
Betoule Marc committed
507 508
            self.pipelist.append(pipe)
            setattr(self, pipe, Web(os.path.expanduser(dbfile),pipe))
509 510 511 512 513 514 515 516 517 518

    def which_sqlfile(self, path_info):
        try:
            pipe = path_info.split('/')[1]
            if pipe in self.pipelist:
                return getattr(self, pipe).db_file
        except KeyError:
            return None
        return None

519 520
    @cherrypy.expose
    def index(self):
Maude Le Jeune's avatar
doc  
Maude Le Jeune committed
521 522 523
        """ Pipeline index. 
        
        """
Betoule Marc's avatar
Betoule Marc committed
524 525
        html = html_tmp + '<h1>Pipeline Index </h1> <div class="list" id="pipelist"><h2>Select a pipeline to browse</h2><ul>'
        for k in self.pipelist:
526
            html += '<li> <a href="/%s/"> %s</a></li>'%(k,k)
Betoule Marc's avatar
Betoule Marc committed
527
        html += '</ul></div></body></html>'
528
        return html
529

Betoule Marc's avatar
Betoule Marc committed
530 531 532 533 534 535 536 537 538 539 540
class MyApp(cherrypy.Application):
    
    def __init__(self, domain, domaindir, script_name="", config=None):
        self.domain = domain
        self.domaindir = domaindir
        root = PipeIndex()
        for pipe, dbfile in cherrypy.config['Pipelines'].iteritems():
            setattr(root, pipe, Web(os.path.expanduser(dbfile)))
        cherrypy.Application.__init__(self, root, script_name, config={'/':{}})

def start(config, config_file):
Betoule Marc's avatar
Betoule Marc committed
541 542 543
    """Start the web server in daemon mode.

    Store the PID in the file specified in config.
544 545 546 547 548

    Parameters
    ----------
    config: dict where internal config is stored
    config_file: string, user config file
Betoule Marc's avatar
Betoule Marc committed
549
    """
Betoule Marc's avatar
Betoule Marc committed
550 551 552
    from cherrypy.process.plugins import Daemonizer, PIDFile
    cherrypy.config.update(config)
    cherrypy.config.update(config_file)
Betoule Marc's avatar
Betoule Marc committed
553
    cherrypy.tree.mount(PipeIndex(),"",config)
Betoule Marc's avatar
Betoule Marc committed
554 555
    d = Daemonizer(cherrypy.engine)
    d.subscribe()
Betoule Marc's avatar
Betoule Marc committed
556
    p = PIDFile(cherrypy.engine, config['global']['server.pidfile'])
Betoule Marc's avatar
Betoule Marc committed
557
    p.subscribe()
Betoule Marc's avatar
Betoule Marc committed
558
    cherrypy.engine.start()
Betoule Marc's avatar
Betoule Marc committed
559

Betoule Marc's avatar
Betoule Marc committed
560
def stop(config, config_file):
Betoule Marc's avatar
Betoule Marc committed
561 562 563
    """Stop the web server by sending the TERM signal to the daemon process.

    Look for the PID in the file specified in config.
564 565 566 567 568

    Parameters
    ----------
    config: dict, where internal config is stored
    config_file: string, user config file
Betoule Marc's avatar
Betoule Marc committed
569
    """
Betoule Marc's avatar
Betoule Marc committed
570
    import signal
571
    cherrypy.config.update(config)
Betoule Marc's avatar
Betoule Marc committed
572 573 574 575 576 577
    cherrypy.config.update(config_file)
    pidf = open(config['global']['server.pidfile'])
    pid = int(pidf.read())
    pidf.close()
    os.kill(pid, signal.SIGTERM)
    
Betoule Marc's avatar
Betoule Marc committed
578 579 580 581
def update_config(pipename, pipepath, config_file):
    """ Add a new Pipeline entry to the config file.

    If the config file does not exist, create a new one.
582 583 584 585 586 587

    Parameters
    ----------
    pipename: string, pipe short name
    pipepath: string, pipe instance root path
    config_file: string, user config file
Betoule Marc's avatar
Betoule Marc committed
588 589 590 591 592 593 594
    """
    import ConfigParser
    c = ConfigParser.ConfigParser()
    c.read(config_file)
    if not c.has_section('Pipelines'):
        c.add_section('Pipelines')
    if os.path.isfile(pipepath):
Marc Betoule's avatar
Marc Betoule committed
595 596
        pipepath = os.path.expanduser(pipepath)
        pipepath = os.path.abspath(pipepath)
Betoule Marc's avatar
Betoule Marc committed
597 598 599 600 601 602 603
        c.set('Pipelines', pipename, '"'+pipepath+'"')
        f=file(config_file,'w')
        c.write(f)
        f.close()
    else:
        print pipepath+ " not found"
    
604
def main():
605
    import optparse
Betoule Marc's avatar
Betoule Marc committed
606
    parser = optparse.OptionParser(usage="\nTo start/stop the web server:\n  %prog start | stop\nTo track a new pipeline under the name <name> with the web interface:\n  %prog track <name> </path/to/sqlfile>")
607 608
    parser.add_option('-H', '--host',
                      help='serve this address',default='0.0.0.0')
Betoule Marc's avatar
Betoule Marc committed
609 610
    parser.add_option('-c', '--config-file',
                      help='Read pipelet configuration from this file',default=os.path.expanduser('~/.pipelet'))
Marc Betoule's avatar
Marc Betoule committed
611
    parser.add_option('-p', '--port',
612
                      help='port the server is listenning to', default=8080, type='int')
Betoule Marc's avatar
Betoule Marc committed
613 614
    parser.add_option('-n', '--no-daemon',
                      help='Run the web server interactively instead of starting a daemon, log errors and access to the screen. Stop with C-c.', default=False, action='store_true')
Betoule Marc's avatar
Betoule Marc committed
615 616
    parser.add_option('-k', '--krenew',
                      help='In deamon mode ask and renew afs token.', default=False, action='store_true')
Betoule Marc's avatar
Betoule Marc committed
617
    parser.add_option('-l', '--error-file',
Betoule Marc's avatar
Betoule Marc committed
618 619
                      help='Logging file for errors when run in daemon mode',
                      default=os.path.expanduser('~/pipelet.errors'))
Betoule Marc's avatar
Betoule Marc committed
620
    parser.add_option('-a', '--access-file',
Betoule Marc's avatar
Betoule Marc committed
621 622
                      help='Logging file for access when run in daemon mode',
                      default=os.path.expanduser('~/pipelet.access'))
Betoule Marc's avatar
Betoule Marc committed
623
    parser.add_option('-i', '--pid-file',
Betoule Marc's avatar
Betoule Marc committed
624 625
                      help='Store the pid in daemon mode',
                      default=os.path.expanduser('~/pipelet.pid'))
626

Betoule Marc's avatar
Betoule Marc committed
627 628
    (options, args) = parser.parse_args()
    
629 630 631 632
    config = {'/static':
                  {'tools.staticdir.on': True,
                   'tools.staticdir.dir': os.path.join(current_dir,'static'),
                   },
633
              'global':{'server.socket_port':options.port, 'server.socket_host':options.host,
Betoule Marc's avatar
Betoule Marc committed
634 635 636 637 638 639
                        'server.thread_pool':10,
                        'log.screen':False, 
                        'log.error_file': options.error_file,
                        'log.access_file':options.access_file,
                        'server.pidfile':options.pid_file}
              }
640
    
Betoule Marc's avatar
Betoule Marc committed
641 642 643 644 645 646 647 648
    if options.no_daemon:
        config['global']['log.screen'] = True
        cherrypy.config.update(config)
        cherrypy.config.update(options.config_file)
        cherrypy.tree.mount(PipeIndex(),"",config)
        cherrypy.engine.start()
        cherrypy.engine.block()
        exit(0)
Betoule Marc's avatar
Betoule Marc committed
649 650 651 652 653 654 655
    if options.krenew:
        cherrypy.config.update(config)
        cherrypy.config.update(options.config_file)
        cherrypy.tree.mount(PipeIndex(),"",config)
        cherrypy.engine.start()
        cherrypy.engine.block()
        exit(0)
Betoule Marc's avatar
Betoule Marc committed
656
    if len(args) < 1:
Betoule Marc's avatar
Betoule Marc committed
657 658 659
        parser.print_usage()
        exit(-1)
    if args[0] == 'start':
Betoule Marc's avatar
Betoule Marc committed
660
        start(config, options.config_file)
Betoule Marc's avatar
Betoule Marc committed
661
    elif args[0] == 'stop':
Betoule Marc's avatar
Betoule Marc committed
662
        stop(config, options.config_file)
Betoule Marc's avatar
Betoule Marc committed
663 664
    elif args[0] == 'restart':
        restart()
Betoule Marc's avatar
Betoule Marc committed
665 666 667 668 669
    elif args[0] == 'track':
        if len(args) != 3:
            parser.print_usage()
            exit(-1)
        update_config(args[1], args[2], options.config_file)
670
    else:
Betoule Marc's avatar
Betoule Marc committed
671 672 673
        parser.print_usage()
        exit(-1)
    
674
    
675 676
if __name__ == "__main__":
    main()