+ gitignore + squelette oubliés

.gitignore

+/build
+/html
+/pipelet.aux
+/pipelet.log
+/pipelet.nav
+/pipelet.out
+/pipelet.pdf
+/pipelet.snm
+/pipelet.toc

pipelet.tex

+\documentclass{beamer}
+\usepackage{graphicx}
+\usepackage{amsmath}
+\usepackage[utf8]{inputenc}
+\usepackage{multicol}
+\usepackage{ulem}
+\usepackage{color}
+\usepackage{listings}
+\usepackage{wasysym}
+\useoutertheme{infolines}
+\usepackage{hangcaption}
+
+\newcommand{\unnumberedcaption}%
+%	{\@dblarg{\@unnumberedcaption\@captype}}
+
+\begin{document}
+
+\begin{frame}
+{Pipelet}
+\tableofcontents
+\end{frame}
+
+\section{Titre}
+\begin{frame}
+
+\end{frame}
+
+\end{document}
\ No newline at end of file

pipelet/.gitignore

+/__init__.pyc
+/pipeline.pyc
+/repository.pyc
+/scheduler.pyc
+/task.pyc
+/tracker.pyc
+/utils.pyc
+/worker.pyc

pipelet/static/main.py

+""" Example main illustrating various ways of launching jobs.
+"""
+
+from pipelet.pipeline import Pipeline
+from pipelet.launchers import launch_interactive, launch_process, launch_pbs
+import os
+import sys
+import logging
+
+
+### Pipeline properties
+
+
+### pipe dot scheme
+### The dependencies between segments must form a directed acyclic
+### graph. This graph is described by a char string using a subset of the
+### graphviz dot language (http://www.graphviz.org). For exemple the string:
+### """
+### a -> b -> d;
+### c -> d;
+### c -> e;
+### """
+### defines a pipeline with 5 segments {"a", "b", "c", "d", "e"}. The
+### relation "a->b" ensures that the processing of the segment "a" will be
+### done before the processing of its child segment "b". Also the output
+### of "a" will be feeded as input for "b". In the given example, the node
+### "d" has two parents "b" and "c". Both will be executed before "d". As
+### their is no relation between "b" and "c" which of the two will be
+### executed first is not defined.
+### See doc for more details. 
+###
+pipe_dot = """ 
+'default'
+"""
+
+### code_dir
+### This path corresponds to the directory where the segment python scripts are found. 
+### One may choose a location which is regularly backed up
+code_dir = './'
+
+### prefix
+### This path corresponds to the directory where the pipeline products will be stored. 
+### One may choose a location with large disk space and no backup. 
+prefix   = './'
+
+
+### log_level
+### Set the log level which will be displayed on stdout. 
+### Logs are also written in pipelet log files with level set to DEBUG.
+### Log files are available from the web interface.  
+log_level = logging.DEBUG
+
+### job_name
+### This string will be used in the PBS mode to tag all pipeline jobs. 
+job_name = 'pipelet'
+
+### job_header
+### This string will be put at the beginning of each PBS jobs. 
+### Error and log files will be available from the interface with 
+### the others log files.  
+job_header = """
+#/bin/bash
+echo $PYTHONPATH
+""" 
+
+### cpu_time
+### Maximum cpu time used in PBS mode. 
+cpu_time = "00:30:00"
+
+
+### print_info
+### Print additional informations after running to access the web interface
+print_info = True
+
+
+
+def main(print_info=print_info):
+    """ Run the pipeline. 
+    """
+
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option('-a', '--add-workers', metavar='N',
+                      help='Submit N supplementary jobs without launching a new server.',type='int')
+    parser.add_option('-d', '--debug',
+                      help='Start jobs in interactive mode',action="store_true",default=False)
+    parser.add_option('-p', '--process', metavar='N',
+                      help='Launch jobs as local parallel processes',type='int')
+    (options, args) = parser.parse_args()
+    
+    
+    ## Build pipeline instance
+    P = Pipeline(pipe_dot, code_dir=code_dir, prefix=prefix, matplotlib=True, matplotlib_interactive=True)
+    
+    ## Interactive mode
+    if options.debug:
+        w, t = launch_interactive(P, log_level=log_level)
+        w.run()
+    ## Process mode
+    elif options.process:
+        launch_process(P, options.process,log_level=log_level)
+    ## PBS mode
+    elif options.add_workers:
+        launch_pbs(P,options.add_workers , address=(os.environ['HOST'],50000), job_name=job_name, cpu_time=cpu_time, job_header=job_header)
+    else:
+        launch_pbs(P, 1, address=(os.environ['HOST'],50000), job_name=job_name, cpu_time=cpu_time, server=True, job_header=job_header, log_level=log_level)
+
+    if print_info:
+        print "1- Run 'pipeweb track <shortname> %s' \n to add the pipe to the web interface.\n"%(P.sqlfile)
+        print "2- Set acl with 'pipeutils -a <username> -l 2 %s'\n"%P.sqlfile
+        print "3- Launch the web server with 'pipeweb start'"
+        print "You should be able to browse the result on the web page http://localhost:8080\n"
+
+if __name__ == "__main__":
+    main()
+    

pipelet/static/seg_default_code.py

+""" This is the docstring of the segment script.
+
+This segment script illustrates how to use the various utilities
+available from the default environment. 
+"""
+
+### How to organize the segment script ?
+###
+### A segment script contains some python code which will be applied on
+### some data set.  
+### The pipelet software has no demand on the code, but it
+### offers a lot of utilities which may set the user straight 
+### about some good practice. 
+###
+### From the pipeline point of view the segment script corresponds to 
+### some processing applied to the data (input/output) with some tuned parameters. 
+### The pipelet software offers some default utilities with respect of that 
+### 3 entities :
+### - processing, 
+### - data, 
+### - parameters. 
+
+### The data and files utilities : 
+###
+### Data are usually stored on files. 
+### The pipelet software offers a specific location for all data files
+### that can be parsed from the web interface. 
+### To get one on those files : 
+output_fn = get_data_fn ('mydata.dat')
+
+### To retrieve some data file from an upstream segment named 'first': 
+input_fn = glob_seg ('*.dat', 'first')[0]
+
+### One may need some temporary files also : 
+temporary_fn = get_tmp_fn()
+
+### To read and write data to files, one may use some default routines
+### based on the pickle python module :
+load_products (input_fn , globals(), ["D"])
+save_products (output_fn, globals(), ["D"])
+
+### The parameters utilities : 
+###
+### The pipelet software saves automatically all parameters listed in
+### the variable named 'var_key'.
+### Some of them can be made visible from the interface using the
+### variable 'var_tag'
+var_key = ['my_dim', 'my_option', 'my_nb_of_iter']
+var_tag = ['my_dim', 'my_nb_of_iter']
+
+### To retrieve some parameters of an upstream segment : 
+load_param ('first', globals(), ['my_previous_option'])
+
+### The processing utilities : 
+###
+### A standard logging.Logger object is available from the segment
+### context.
+log.info("Until now, nothing's wrong\n")
+
+### It may occur that the processing applied to the data contains some
+### generic code you want to recycle from one segment to another.
+### This portion of code can be factorized in a hook script names
+### 'seg_segname_hookname.py' which can be called with: 
+hook ('preproc', globals())
+
+### It is also possible to call an arbitrary subprocess without
+### loosing the log facility: 
+logged_subprocess(['my_sub_process', 'my_dim', 'my_nb_of_iter'])
+
+
+### The pipe scheme utilities : 
+###
+### The pipe scheme can be controlled from the segment environment
+### also.  The default behaviour is set by the segment input and
+### output variables : 
+output = [1,2,3] ## will generate 3 instances of the downstream
+                 ## segment. Each of them will receive one element of
+                 ## the list as input.
+output = input   ## will generate 1 instance of the downstream
+                 ## segment, per current instance.
+output = None    ## will generate 1 instance of the downstream
+                 ## segment for all current instance. 
+
+### This default behavious can be altered by specifying an @multiplex
+### directive (see documentation). 
+

test/.gitignore

+/.sqlstatus