Commit 43d228f3 authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

new demo cmb pipe

parent e293ef8f
......@@ -35,12 +35,12 @@
\usepackage{wasysym}
\useoutertheme{infolines}
\usepackage{hangcaption}
\newcommand{\pipelet}{\textbf{\small{PIPELET}}\xspace}
\newcommand{\pipelet}{\textbf{\scriptsize{PIPELET}}\xspace}
\title[Pipelet]{The \pipelet software}
\author[Betoule, Le Jeune]{ Maude \textsc{Le Jeune},Marc \textsc{Betoule}}
\institute[LPNHE,APC]{}
\date[2010/11/26]{november, 26th, 2010}
\title[]{The \pipelet software (1.0)}
\author[Pipelet]{ Maude \textsc{Le Jeune}, Marc \textsc{Betoule}}
\institute[v1.0]{}
\date[2010/11/26]{November, 26th, 2010}
\newcommand{\unnumberedcaption}%
% {\@dblarg{\@unnumberedcaption\@captype}}
......@@ -74,7 +74,7 @@ Usually in scientific data processing:
\end{figure}
The \pipelet software answers the 3 above items:
\begin{itemize}
\item Native parallelization and CPU time sparing
\item Native parallelisation and CPU time sparing
\item Guarranty traceability
\item Offer comparison facilities
\end{itemize}
......@@ -120,8 +120,7 @@ The \pipelet software answers the 3 above items:
\includegraphics[width=0.75\textwidth]{pipelet_scheme_small3.pdf}
\end{figure}
\end{itemize}
\hspace{0.5cm}$\rhd$ The pipe scheme is defined by user (reflecting actual dependency
scheme, or not). Any directed acyclic graph allowed. \\
\hspace{0.5cm}$\rhd$ The pipe scheme is defined by user. Any directed acyclic graph allowed. \\
\hspace{0.5cm}$\rhd$ Filenames are provided by the \pipelet engine. \\
\end{frame}
......@@ -133,14 +132,17 @@ The \pipelet software answers the 3 above items:
\begin{itemize}
\item \pipelet segments are written in Python.
\item Native parallelisation scheme applied on data to process (\textbf{tasks}).
\item \textbf{Tasks} can be executed in different modes (sequential
\item Tasks can be executed in different modes (sequential
for debugging, using process on smp machine, batch mode on
clusters).
\end{itemize}
\begin{figure}
\includegraphics[width=0.50\textwidth]{parral.pdf}
\end{figure}
\hspace{0.5cm}$\rhd$ Python interfaces nicelly with
other languages : C/C++ extensions or SWIG/Boost wrapping. \\
\hspace{0.5cm}$\rhd$ Parallelisation at the highest level. No need to
learn OpenMP/MPI. Combination of distributed/shared memory //tion eased. \\
learn OpenMP/MPI. \\
\hspace{0.5cm}$\rhd$ Portability offered by the different running
modes (\textbf{launchers}).
\end{frame}
......@@ -152,14 +154,17 @@ modes (\textbf{launchers}).
reused by someone else, but pipelines are always trashed !}
\end{enumerate}
\begin{itemize}
\item Segments can be documented using Python docstrings syntax.
\item Pipeline scheme written and displayed using dot utility.
\item Segments can be documented using Python docstring syntax.
\item Pipeline scheme written and displayed using graphviz dot language.
\item Collaborative work eased by a web interface and \textit{code
repositories}.
\end{itemize}
\hspace{0.5cm}$\rhd$ \\
\hspace{0.5cm}$\rhd$ \\
\hspace{0.5cm}$\rhd$ \\
\begin{figure}
\includegraphics[width=0.25\textwidth]{p.pdf}
\includegraphics[width=0.15\textwidth]{p2.pdf}
\end{figure}
\hspace{0.5cm}$\rhd$ Non trivial dependency scheme are easy to read. \\
\hspace{0.5cm}$\rhd$ Results can be accessed by different users with a minimum amount of indications (tags, docstrings).\\
\end{frame}
\section{How it works}
......@@ -185,8 +190,8 @@ modes (\textbf{launchers}).
\begin{itemize}
\item \verb pipedot is the string description of the pipeline
\begin{verbatim}pipedot = """
1->2->4
3->4
1->2->4;
3->4;
"""
\end{verbatim}
\item \verb codedir is the path of the processing code files (.py)
......@@ -210,7 +215,7 @@ modes (\textbf{launchers}).
\begin{itemize}
\item It benefits from an improved namespace to:
\begin{itemize}
\item control the pipe parallelization scheme;
\item control the pipe parallelisation scheme;
\item save and load I/O's and provide filenames;
\item save and load parameters;
\item execute or include subprocess
......@@ -259,20 +264,38 @@ $\bullet$ Download/visualize/delete product files & from the product page\\
\begin{frame}
{Problematic}
Bottlenecks
One wants to perform some spectral estimation using inverse noise
weighting on simulated data. \\
The wish list is:
\begin{itemize}
\item mll
\item MC
\item abstraction : masque, correcteur
\item Design a prototype pipeline to get a first result.
\item Test different noise levels and weighting masks.
\item Spare coupling matrix computation as much as possible.
\item Perform Monte Carlo studies to get some error bars.
\end{itemize}
\begin{figure}
\includegraphics[width=0.35\textwidth]{map.png}
\hspace{1cm}
\includegraphics[width=0.3\textwidth]{cl.jpg}
\end{figure}
\end{frame}
\subsection{A pipelet solution}
\begin{frame}
\begin{frame}[fragile]
{A \pipelet solution}
\begin{columns}\begin{column}{0.6\textwidth}
$\bullet$ \verb!cmb.py!:\\ \hspace{0.2cm}generate cmb map\textcolor{blue}{(s)} from lambdaCDM cls.\\
$\bullet$ \verb!noise.py!:\\ \hspace{0.2cm}generate noise map\textcolor{blue}{(s)} from hitcount map and compute coupling matrix.\\
$\bullet$ \verb!clcmb.py!:\\ \hspace{0.2cm}compute cls of the noisy cmb map\textcolor{blue}{(s)} and correct it from coupling matrix.\\
$\bullet$ \verb!clnoise.py!:\\ \hspace{0.2cm}compute cls of the noise map. \\
$\bullet$ \verb!clplot.py!:\\ \hspace{0.2cm}make a plot of the cls estimates \textcolor{blue}{(with error bars)}.\\
\end{column}
\begin{column}{0.4\textwidth}
\begin{figure}
\includegraphics[width=1\textwidth]{pcmb.pdf}
\end{figure}
\end{column}
\end{columns}
\end{frame}
\subsection{Zooming the code}
......@@ -284,11 +307,26 @@ Bottlenecks
\subsection{Browsing the result}
\subsection{Deployment}
\begin{frame}
\begin{frame}[fragile]
{Deployment}
Laptop
Cluster Adamis
CCin2p3/Magique
\begin{enumerate}
\item Development phase : on your laptop.
\begin{itemize}
\item run the pipe using the python debugger \verb!%pdb!
\item and the \pipelet interactive mode (sequential)
\end{itemize}
\item Production phase : on a desktop machine / adamis cluster.
\begin{itemize}
\item use the process mode to dispatch tasks between cores.
\item use the batch mode to dispatch tasks between cores and nodes.
\end{itemize}
\item Release phase : @ CCin2p3 or Magique3.
\begin{itemize}
\item use a cutosmized segment environment (DMC database)
\item use a customized batch launcher (BQS)\\
(Those 2 customizations are already done.)
\end{itemize}
\end{enumerate}
\end{frame}
......@@ -309,6 +347,7 @@ $\rhd$ Download from \url{http://gitorious.org/pipelet}
\end{itemize}
\vspace{0.5cm}
$\rhd$ Features and bugs are tracked from the IN2P3 forge.\\
\hspace{0.3cm}Don't hesitate to subscribe the \pipelet project to give your feedback and follow the project news.\\
\vspace{0.5cm}
$\rhd$ CMB demo pipeline under : \verb!pipelet/test/cmb!
\end{frame}
......
""" clcmb.py
Coadd cmb and noise map.
Compute the power spectrum
Correct it from the mode coupling matrix
"""
import healpy as hp
import pylab as pl
import spherelib as sp
### Retrieve some global parameters
load_param("cmb", globals(), ["lmax", "cmb_unit"])
load_param("noise", globals(), ["noise_unit", "noise_freq"])
#multiplex cross_prod group_by 'cmb'
seed = seg_input["cmb"]
### Get inputs
cmb_fn = glob_seg("cmb", "map*.fits")[0]
cmb_map = hp.read_map(cmb_fn)
noise_fn = glob_seg("noise", "map*.fits")[0]
noise = hp.read_map(noise_fn)
cv_factor= sp.astro.convfact(freq=noise_freq, from=noise_unit, to=cmb_unit)
noise = noise*cv_factor
### Mask the input map
cmb_map = cmb_map + noise
### Compute power spectrum
cl = hp.anafast (cmb_map, lmax)
### save it to disk
cl_fn = get_data_fn("cls.txt")
pl.savetxt(cl_fn, cl)
## load mll'
mll_fn = glob_seg('noise', 'mll*')[0]
load_products (mll_fn , globals(), ["mll"])
## correct cls
cl = pl.linalg.solve(mll , cl)
### save it to disk
cl_fn = get_data_fn("cls-mll.txt")
pl.savetxt(cl_fn, cl)
### Set output
seg_output = [seed]
""" clplot.py
Correct cls from mask effect.
Make a plot.
"""
......@@ -8,31 +7,16 @@ import healpy as hp
import pylab as pl
import spherelib as sp
## Gather all cls
#multiplex cross_prod group_by 0
load_param('mask', globals(), ["mask_types", "lmax", "fsky"])
load_param('cmb', globals(), ["input_cl"])
## load cls
apo_cls= pl.loadtxt(glob_seg('cls', 'apodized*.txt')[0])
bin_cls= pl.loadtxt(glob_seg('cls', 'binary*.txt')[0])
## load mll'
mll_fn = glob_seg('mask', 'mll*')[0]
load_products (mll_fn , globals(), ["mll"])
## corrected cls
binc_cls = pl.linalg.solve(mll , bin_cls)
bin_cls = bin_cls/fsky
apo_cls = apo_cls/fsky
cl_cmb = pl.loadtxt(glob_seg('clcmb', 'cl*.txt')[0])
cl_mll = pl.loadtxt(glob_seg('clcmb', 'cl*mll*.txt')[0])
## smoothed cls
dloverl = 0.05
binc_cls = sp.smooth_cl(binc_cls, dloverl=dloverl)
bin_cls = sp.smooth_cl(bin_cls, dloverl=dloverl)
apo_cls = sp.smooth_cl(apo_cls, dloverl=dloverl)
bin_cl_cmb = sp.smooth_cl(cl_cmb, dloverl=dloverl)
bin_cl_mll = sp.smooth_cl(cl_mll, dloverl=dloverl)
## input cls
inp_cls = pl.loadtxt(input_cl)[0:lmax+1,0]
......@@ -40,10 +24,9 @@ inp_cls = pl.loadtxt(input_cl)[0:lmax+1,0]
## make a plot
ell = pl.arange(lmax+1)
pl.figure()
pl.plot(ell, ell*(ell+1)*bin_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*binc_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*apo_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*bin_cl_cmb/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*bin_cl_mll/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*inp_cls/(2*pl.pi), color="black")
pl.legend([ "fsky", "mll", "apodized mask", "input"], loc="upper right")
pl.legend([ "empirical", "mll", "input"], loc="upper right")
pl.savefig(get_data_fn("cls.png"))
pl.savefig(get_data_fn("cls.pdf"))
""" clplot.py
Correct cls from mask effect.
Make a plot.
"""
import healpy as hp
import pylab as pl
import spherelib as sp
## Gather all cls
#multiplex cross_prod group_by 0
load_param('mask', globals(), ["mask_types", "lmax", "fsky"])
load_param('cmb', globals(), ["input_cl"])
## load cls
apo_cls= pl.loadtxt(glob_seg('cls', 'apodized*.txt')[0])
bin_cls= pl.loadtxt(glob_seg('cls', 'binary*.txt')[0])
## load mll'
mll_fn = glob_seg('mask', 'mll*')[0]
load_products (mll_fn , globals(), ["mll"])
## corrected cls
binc_cls = pl.linalg.solve(mll , bin_cls)
bin_cls = bin_cls/fsky
apo_cls = apo_cls/fsky
## smoothed cls
dloverl = 0.05
binc_cls = sp.smooth_cl(binc_cls, dloverl=dloverl)
bin_cls = sp.smooth_cl(bin_cls, dloverl=dloverl)
apo_cls = sp.smooth_cl(apo_cls, dloverl=dloverl)
## input cls
inp_cls = pl.loadtxt(input_cl)[0:lmax+1,0]
## make a plot
ell = pl.arange(lmax+1)
pl.figure()
pl.plot(ell, ell*(ell+1)*bin_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*binc_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*apo_cls/(2*pl.pi))
pl.plot(ell, ell*(ell+1)*inp_cls/(2*pl.pi), color="black")
pl.legend([ "fsky", "mll", "apodized mask", "input"], loc="upper right")
pl.savefig(get_data_fn("cls.png"))
pl.savefig(get_data_fn("cls.pdf"))
......@@ -11,9 +11,9 @@ lst_par = ['lmax', 'nside', 'cmb_unit', 'seed', 'input_cl']
lst_tag = lst_par[:-2]
nside = seg_input.values()[0]
seed = seg_input.values()[1] ## actually not use by synfast
lmax = 2*nside
cmb_unit = "uK_CMB"
seed = 0 ## actually not use by synfast
### Generate a cmb map
......
......@@ -32,10 +32,12 @@ import os.path as path
###
nside = 512
seeds = [1]
pipe_dot = """
cmb->cls->clplot;
mask->cls;
cmb->clcmb->clplot;
noise->clcmb;
noise->clnoise->clplot;
"""
### code_dir
......@@ -96,7 +98,9 @@ def main(print_info=print_info):
## Build pipeline instance
P = Pipeline(pipe_dot, code_dir=code_dir, prefix=prefix, matplotlib=True, matplotlib_interactive=True)
P.push(cmb=[nside], mask=[nside])
for seed in seeds:
P.push(cmb=[(nside,seed)])
P.push(mask=[(nside,seed)])
## Interactive mode
if options.debug:
......
""" noise.py
Generate a noise map from hitcount map.
Compute coupling matrix
"""
import healpy as hp
import pylab as pl
### Define some global parameters
lst_par = ['lmax', 'nside', 'noise_unit', 'seed', 'input_hitcount', 'noise_level']
lst_tag = lst_par[:-2]
nside = seg_input.values()[0]
seed = seg_input.values()[1] ## actually not use by synfast
lmax = 2*nside
noise_unit = "K_RJ"
### Generate a noise map
input_hitcount = ""
hitcount = hp.read_map(input_hitcount)
hitcount = hp.ud_grade(hitcount, nside)
hitnside = pl.sqrt(len(hitcount)/12)
hitcount = hitcount * (hitnside/nside)**2
noise_map= pl.random.standard_normal(12*nside*nside)*noise_level / pl.sqrt(hitcount)
### Save it to disk
noise_fn = get_data_fn("map_noise.fits")
hp.write_map(noise_fn, noise_map)
### Compute mll'
mll = sp.mask2mll (hitcount, lmax)
mll_fn = get_data_fn("mll.dat")
save_products (mll_fn, globals(), ["mll"])
### Make a plot
noise_fig = noise_fn.replace('.fits', '.png')
hp.mollview(noise_map)
pl.savefig (noise_fig)
### Set output
seg_output = [seed]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment