Commit 79a209db authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Updates new yam conf

parent 621d9e85
......@@ -68,7 +68,6 @@ $ stream_create_base_config
--pixel_order ./LST-1.1.Run03944.0000_pix_order_bin.npy
--gain_pedestal ./calibration_Run03945.h5
--output_file base_structure_hdf5_3945.h5
````
- `--config_file` Once `HiPeRTA` is installed, it must be on
......@@ -110,7 +109,7 @@ HDF5 file size: 451.0KiB
You can also plot the calibration distributions to check that everything worked correctly.
```bash
stream_plot_r0_calib -i calibration_Run03945.h5
$ stream_plot_r0_calib -i calibration_Run03945.h5
```
......
obs_id: 0
run_id: 0
tel_id: 1
host_config:
- host:
name: localhost
port: 3390
nb_thread: 2
slurm_nodelist: 'cp52'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3391
nb_thread: 2
slurm_nodelist: 'cp52'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3392
nb_thread: 2
slurm_nodelist: 'cp53'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3393
nb_thread: 2
slurm_nodelist: 'cp53'
slurm_queue: 'short'
slurm_account: 'aswg'
base_filenames: # 'dlX_obsId_runId_telId_bunchId.h5'
dl1: 'dl1'
dl2: 'dl2'
dl3: 'dl3'
slurm_config: # ABS paths
dl1: './dl1'
dl2: './dl2'
dl3: './dl3'
extraargumentdl1dl2: '-p PATH_models' # Input file already passed
extraargumentdl2dl3: 'change/script/arguments' # Et pum, chocapic ! TODO
confighdf5: 'path/to/filename'
nbEventPerHdf5: 10000
# Standard configuration for offline and online analysis
threshold_bad_adc: 4094
threshold_bad_ped: 100000
# Integration (GLOBAL_PEAK_INTEGRATOR, NEIGHBOUR_PEAK_INTEGRATOR, LOCAL_PEAK_INTEGRATOR)
integration_type: GLOBAL_PEAK_INTEGRATOR
nb_slice_before_peak: 3
nb_slice_window: 7
# Useful on real data. Set to 3 and 2, respectively, for LST-1
nb_first_slice_to_reject: 0
nb_last_slice_to_reject: 0
# Cut on calibrated/integrated signal
hillas_threshold_signal_tel: 30
# Maximum integrated signal in the camera (60000 is fine to remove the flatfield inside real data)
hillas_threshold_signal_tel_max: 1000000
# Cleaning configuration (TAILCUT, WAVELET)
cleaning_type: TAILCUT
hillas_threshold_center: 6
hillas_threshold_neighbours: 3
min_number_neighbour_pixels: 1
keep_isolated_pixel: false
wavelet_threshold: 4
min_selected_pixel: 2
##################################################
# Stream configuration for hiperta_stream_r0_dl1 #
##################################################
# Offset in bytes of information in the zfits stream. To decode DAQ R0 stream
zfits_offset:
offset_event_id: 58
offset_time_second: 76
offset_time_qns: 81
offset_event_type: 86
offset_waveform_high: 100
offset_waveform_low: 148500
# Used by hiperta_stream_r0_dl1
nb_event_average_ped: 10000
nb_event_per_hdf5: 10000
keep_rejected_event: false
stream_read_zfit: false
# Info to be passed also to start_hiperta_stream.py
obs_id: 42
run_id: 3945
tel_id: 1
debug_performance:
nbMaxMessage: -1
config_hiperta: 'path/to/hiperta_config' # contains config for r0_to_dl1
config_lstchain: 'path/to/lstchain_config' # contains config for dl1_to_dl2 and dl2_to_dl3
#hiperta_stream_r0_dl1 -h
#Usage :
# hiperta_stream_r0_dl1 --input=file.pslicetel --configcut=configcut.txt --refshape=refshape.ppixelrefshape
# hiperta_stream_r0_dl1 -i file.pslicetel -c configcut.txt -r refshape.ppixelrefshape
#Parameters :
# -i FILE1 FILE2 FILEN --input=FILENAME : input file name (default localhost)
# -p UINT --port=UINT : port to be used (default : 3390)
# -o BASEFILENAME --output=BASEFILENAME : base of the output dl1 filename in HDF5 format (default 'dl1_')
# -f FILENAME --confighdf5=FILENAME : base hdf5 file which contains all the gain/pedestal/refshape/pixel position/pixel order to be put in the output dl1
# -b UINT --baseoutputfilevalue=UINT : base value for renaming the hillas output file (default 0)
# -e UINT --nbEventPerHdf5=UINT : number of events in the created HDF5 files (default 100)
# -m UINT --nbMaxMessage=UINT : total number of events (default 1000)
# -c FILENAME --configcut=FILENAME : all cuts configuration
# -d --default : dump default cuts configuration example
# -n UINT --nbprocess=UINT : total number of process launch on the same program
# -g UINT --processindex=UINT : index of the process (from 0 to nbprocess - 1)
# -t UINT --nbthread=UINT : number of thread to be used to run the analysis
#Arguments to be passed to initialise the SubJobConfig to start the rest of the analysis
# --nodelist=STRING : list of nodes where to slurm is able to start the RTA sub-jobs
# --queuename=STRING : name of the queue to be used to with slurm to start the RTA sub-job (default 'short')s
# --dirdl1=DIRNAME : directory where to write DL1 files (default '$PWD/dl1')
# --dirdl2=DIRNAME : directory where to write DL2 files (default '$PWD/dl2')
# --dirdl3=DIRNAME : directory where to write DL3 files (default '$PWD/dl3')
# --extraargumentdl1dl2=STRING : extra argument to use DL1->DL2 sub-jobs (such as model)
# --extraargumentdl2dl3=STRING : extra argument to use DL2->DL3 sub-jobs (such as some cuts)
nbMaxMessage: 500000
# Common path to save the output HDF5 files
common_abs_path: '/fefs/aswg/workspace/enrique.garcia/RTA/second_night'
base_prefix:
prefix_dl1: 'dl1'
prefix_dl2: 'dl2'
prefix_dl3: 'dl3'
exchange_dir: # ABSOLUTE PATHS !!!
exchange_dl1: '${common_abs_path}/dl1' # '/dev/shm'
exchange_dl2: '${common_abs_path}/dl2' # '/dev/shm'
exchange_dl3: '${common_abs_path}/dl3'
confighdf5: '${common_abs_path}/base_structure_hdf5_3945.h5' # ABS PATH
tmp_dl1_file: "${exchange_dl1}/${prefix_dl1}_${threadFileId}.h5"
tmp_dl2_file: "${exchange_dl2}/${prefix_dl2}_${threadFileId}.h5"
tmp_dl3_file: "${exchange_dl3}/${prefix_dl3}_${threadFileId}.h5"
model_path: "/fefs/aswg/workspace/thomas.vuillaume/mchdf5/models/20200629_prod5_trans_80/zenith_20deg/south_pointing/20210428_vRTA400-dev_v0.6.3_prod5_trans_80_local_tail_8_4"
lstchain_config_file: "/fefs/aswg/workspace/enrique.garcia/RTA/lstchain_standard_config_v063-RTA_Tel1_wo_n_islands.json"
# Job SLURM hiperta_stream_r0_dl1. slurm_nodelist, slurm_queue, slurm_account ARE NEEDED !
host_config: # This should be repeated the X number of threads to be launched.
- host:
name: cp01
port: 3390
nb_thread: 1
slurm_nodelist: 'cp42'
slurm_queue: 'short'
slurm_account: 'aswg'
# Configuration to start slurm sub-jobs automatically from hiperta_stream_r0_dl1 when a DL1 file is produced
job:
job_argument: "-A aswg -p short"
machines: "cp42-cp49"
dl1dl2:
command: "hiperta_dl1_to_dl2 -f ${tmp_dl1_file} -o ${tmp_dl2_file} -p ${model_path} -c ${lstchain_config_file}"
machines: "cp42-cp49"
plotdl1:
command: "hiperta_plot_dl1 -i ${tmp_dl1_file}"
job_argument: "-A aswg -p short"
machines: "cp42-cp49"
rmdl1:
command: "rm ${tmp_dl1_file}"
depdendencies: [dl1dl2, plotdl1]
dl2dl3:
command: "hiperta_dl2_dl3 -i ${tmp_dl2_file} -o ${tmp_dl3_file}"
machines: "cp42-cp49"
depdendencies: [dl1dl2]
plotdl2:
command: "hiperta_plot_dl2 -i ${tmp_dl2_file}"
job_argument: "-A aswg -p short"
machines: "cp50-cp54"
depdendencies: [dl1dl2]
rmdl2:
command: "rm ${tmp_dl2_file}"
depdendencies: [dl2dl3, plotdl2]
obs_id: 0
run_id: 0
tel_id: 1
host_config:
- host:
name: localhost
port: 3390
nb_thread: 2
slurm_nodelist: 'cp52'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3391
nb_thread: 2
slurm_nodelist: 'cp52'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3392
nb_thread: 2
slurm_nodelist: 'cp53'
slurm_queue: 'short'
slurm_account: 'aswg'
- host:
name: localhost
port: 3393
nb_thread: 2
slurm_nodelist: 'cp53'
slurm_queue: 'short'
slurm_account: 'aswg'
base_filenames: # 'dlX_obsId_runId_telId_bunchId.h5'
dl1: 'dl1'
dl2: 'dl2'
dl3: 'dl3'
slurm_config: # ABS paths
dl1: './dl1'
dl2: './dl2'
dl3: './dl3'
extraargumentdl1dl2: '-p PATH_models' # Input file already passed
extraargumentdl2dl3: 'change/script/arguments' # Et pum, chocapic ! TODO
confighdf5: 'path/to/filename'
nbEventPerHdf5: 10000
stream_read_zfit: false
debug_performance:
nbMaxMessage: -1
config_hiperta: 'path/to/hiperta_config' # contains config for r0_to_dl1
config_lstchain: 'path/to/lstchain_config' # contains config for dl1_to_dl2 and dl2_to_dl3
#hiperta_stream_r0_dl1 -h
#Usage :
# hiperta_stream_r0_dl1 --input=file.pslicetel --configcut=configcut.txt --refshape=refshape.ppixelrefshape
# hiperta_stream_r0_dl1 -i file.pslicetel -c configcut.txt -r refshape.ppixelrefshape
#Parameters :
# -i FILE1 FILE2 FILEN --input=FILENAME : input file name (default localhost)
# -p UINT --port=UINT : port to be used (default : 3390)
# -o BASEFILENAME --output=BASEFILENAME : base of the output dl1 filename in HDF5 format (default 'dl1_')
# -f FILENAME --confighdf5=FILENAME : base hdf5 file which contains all the gain/pedestal/refshape/pixel position/pixel order to be put in the output dl1
# -b UINT --baseoutputfilevalue=UINT : base value for renaming the hillas output file (default 0)
# -e UINT --nbEventPerHdf5=UINT : number of events in the created HDF5 files (default 100)
# -m UINT --nbMaxMessage=UINT : total number of events (default 1000)
# -c FILENAME --configcut=FILENAME : all cuts configuration
# -d --default : dump default cuts configuration example
# -n UINT --nbprocess=UINT : total number of process launch on the same program
# -g UINT --processindex=UINT : index of the process (from 0 to nbprocess - 1)
# -t UINT --nbthread=UINT : number of thread to be used to run the analysis
#Arguments to be passed to initialise the SubJobConfig to start the rest of the analysis
# --nodelist=STRING : list of nodes where to slurm is able to start the RTA sub-jobs
# --queuename=STRING : name of the queue to be used to with slurm to start the RTA sub-job (default 'short')s
# --dirdl1=DIRNAME : directory where to write DL1 files (default '$PWD/dl1')
# --dirdl2=DIRNAME : directory where to write DL2 files (default '$PWD/dl2')
# --dirdl3=DIRNAME : directory where to write DL3 files (default '$PWD/dl3')
# --extraargumentdl1dl2=STRING : extra argument to use DL1->DL2 sub-jobs (such as model)
# --extraargumentdl2dl3=STRING : extra argument to use DL2->DL3 sub-jobs (such as some cuts)
......@@ -2,6 +2,7 @@
# P. Aubert & E. Garcia - Nov 2020, lock-down Chap. II
import os
import sys
import time
import yaml
import shutil
......@@ -33,7 +34,7 @@ def read_config(file):
exit(-1)
def start_stream_host(host_config, config, host_index, nb_total_threads, debug, log_directory, base_output_file_index):
def start_stream_host(host_config, config, config_file, host_index, nb_total_threads, debug, log_directory, base_output_file_index):
"""
From the just read config file, it creates the final command to be send to slurm to batch/run interactively
the hiperta_stream_r0_dl1 processes.
......@@ -43,7 +44,9 @@ def start_stream_host(host_config, config, host_index, nb_total_threads, debug,
host_config : dict
Pointer to config['host_config']
config : dict
Full configuration loaded into a dictionary
Full PARSED ! configuration loaded into a dictionary
config_file : str
Path to stream_start configuration file
host_index : int
Index of the host to be run. Usually 4 in total, from 0-3.
nb_total_threads: int
......@@ -60,41 +63,34 @@ def start_stream_host(host_config, config, host_index, nb_total_threads, debug,
log : dict
Dictionary to keep the log; {job_id : slurm batch command sent}
"""
# TODO var replacement in config. Only vars that are passed to hiperta_stream_r0_dl1.
# The rest is HiPeRTA who manages it
if config['common_abs_path'] is None:
print(f'Key entry `common_abs_path` needs to be defined !')
sys.exit(-1)
else:
config['exchange_dir']['exchange_dl1'] = \
config['exchange_dir']['exchange_dl1'].replace('${common_abs_path}', config['common_abs_path'])
host_name = host_config['name']
dl1_dir = os.path.abspath(config['exchange_dir']['exchange_dl1'])
output_base_filename = dl1_dir + '/' + config['base_prefix']['prefix_dl1'] + '_' + str(config['obs_id']) + '_' \
+ str(config['run_id']) + '_' + str(config['tel_id']) + '_'
host_port = host_config['port']
base_filename = config['base_filenames']['dl1'] + '_' + str(config['obs_id']) + '_' + str(config['run_id']) + '_' \
+ str(config['tel_id']) + '_'
hdf5config = config['confighdf5']
nb_events = config['nbEventPerHdf5']
queue_name = host_config['slurm_queue']
nb_events = config['nb_event_per_hdf5']
nb_max_events = config['debug_performance']['nbMaxMessage']
nb_threads = host_config['nb_thread']
node_list = host_config['slurm_nodelist']
queue_name = host_config['slurm_queue']
queue_account = host_config['slurm_account']
dl1_dir = os.path.abspath(config['slurm_config']['dl1'])
dl2_dir = os.path.abspath(config['slurm_config']['dl2'])
dl3_dir = os.path.abspath(config['slurm_config']['dl3'])
hiperta_config_cuts = config['config_hiperta']
if config['stream_read_zfit']:
read_zfits = ' --zfits'
read_zfits = ' --zfits --pedperslice'
else:
read_zfits = ''
lstchain_config = config['config_lstchain']
if lstchain_config is not None or lstchain_config != "":
extra_args_dl1_dl2 = config['slurm_config']['extraargumentdl1dl2'] + ' -c ' + lstchain_config
extra_args_dl2_dl3 = config['slurm_config']['extraargumentdl2dl3'] + ' -c ' + lstchain_config
else:
print("\nSpecify a configuration for lstchain, otherwise lstchain will used a standard configuration that"
"most probably will not match with the trained model to be used. ")
exit(-1)
base_cmd = f"""hiperta_stream_r0_dl1 -i {host_name} -p {host_port} -o {base_filename} -f {hdf5config} """ \
f"""-b {base_output_file_index} -e {nb_events} -m {nb_max_events} -c {hiperta_config_cuts} -n {nb_total_threads} """ \
f"""-g {host_index} -t {nb_threads} --nodelist {node_list} --queuename {queue_name} """ \
f"""--queueaccount {queue_account} --dirdl1 {dl1_dir} --dirdl2 {dl2_dir} --dirdl3 {dl3_dir} """ \
f"""--extraargumentdl1dl2='{extra_args_dl1_dl2}' """ \
f"""--extraargumentdl2dl3='{extra_args_dl2_dl3}' {read_zfits} """
base_cmd = f"""hiperta_stream_r0_dl1 -i {host_name} -p {host_port} -o {output_base_filename} """ \
f"""-b {base_output_file_index} -e {nb_events} -m {nb_max_events} -c {config_file} -n {nb_total_threads} """ \
f"""-g {host_index} -t {nb_threads} {read_zfits} """
# Update base_output_file_index
base_output_file_index += nb_threads
......@@ -178,19 +174,17 @@ def main():
configuration = read_config(args.config_file)
host_configuration = configuration['host_config']
num_total_host = len(host_configuration)
if args.debug_mode: # take a single thread for debug
host_configuration = [configuration['host_config'][0]]
num_total_host = 1
stream_jobs = {}
outdirs = configuration['slurm_config']
outdirs = configuration['exchange_dir']
os.makedirs(outdirs['dl1'], exist_ok=True)
os.makedirs(outdirs['dl2'], exist_ok=True)
os.makedirs(outdirs['dl3'], exist_ok=True)
os.makedirs(args.log_directory, exist_ok=True)
os.makedirs(outdirs['exchange_dl1'], exist_ok=True)
os.makedirs(outdirs['exchange_dl2'], exist_ok=True)
os.makedirs(outdirs['exchange_dl3'], exist_ok=True)
os.makedirs(os.path.abspath(args.log_directory), exist_ok=True)
base_output_file_index = 0
nb_total_thread = compute_nb_total_thread(configuration)
......@@ -200,6 +194,7 @@ def main():
jid, base_output_file_index = start_stream_host(
host['host'],
configuration,
args.config_file,
i,
nb_total_thread,
args.debug_mode,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment