Commit 5940cad3 authored by Marc-Antoine Drouin's avatar Marc-Antoine Drouin
Browse files

improve raw file checking

parent 8b3af821
......@@ -144,6 +144,18 @@ RAW2L1_LOGS=${REP_SOURCES}"/logs/raw2l1.log"
NCKS="/usr/bin/ncks"
# CL parameters
# ----------------------------------------------------------------------------
# number of vertical gates
declare -A CL_N_GATES
CL_N_GATES["1"]=770
CL_N_GATES["2"]=385
CL_N_GATES["3"]=1500
CL_N_GATES["4"]=770
CL_N_GATES["6"]=1540
# number of characters per gate
N_CHAR_GATES=5
#------------------------------------------------------------------------------
# verification des parametres
#------------------------------------------------------------------------------
......@@ -152,7 +164,7 @@ if [[ $ERREUR != 0 ]]; then
exit 1
fi
if [[ $fich_type != "SIRTA" && $fich_type != "MF" && $fich_type != "P7" && $fich_type != "ROISSY_5M_30S" && $fich_type != "ROISSY_10M_30S" ]]; then
if [[ $fich_type != "SIRTA" && $fich_type != "MF" ]]; then
echo "Le type de fichier doit etre SIRTA, MF ou P7"
exit 1
fi
......@@ -244,82 +256,44 @@ for file_ in "${input_files[@]}"; do
tmp_file_=$(basename "$file_")
cl31_cor="$tmp_check/${tmp_file_%.*}.cor"
# get vertical resolution of data
vertical_conf=$(sed -nr '0,/CL/s/.*CL.*([0-9]).*/\1/p' "$file_")
n_gates=${CL_N_GATES[$vertical_conf]}
prof_n_chars=$((n_gates * N_CHAR_GATES))
prof_n_chars_p1=$((prof_n_chars + 1))
case $fich_type in
"SIRTA")
awk --posix -v cl31_cor="$cl31_cor" '\
awk --posix -v cl31_cor="$cl31_cor" -v prof_n_chars="$prof_n_chars" -v prof_n_chars_p1="$prof_n_chars_p1" '\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
NR<3 {print $0 >> cl31_cor;next}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}.$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^[\001]CL020521[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==1 {if ($0 ~ /^[[:cntrl:]]*$/ || $0 ~ "File") {next}}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}.?$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^[\001]CL.{6}[\002].?$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == 3851) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == prof_n_chars || length($0) == prof_n_chars_p1) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if ($0 ~ /^[\003][0-9a-z]{4}[\004].$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}' "$file_"
;;
"MF")
awk --posix -v cl31_cor="$cl31_cor" '\
awk --posix -v cl31_cor="$cl31_cor" -v prof_n_chars="$prof_n_chars" -v prof_n_chars_p1="$prof_n_chars_p1" '\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
l==1 {if ($0 ~ /^[\001]$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if ($0 ~ /^ CL120521[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if ($0 ~ /^ CL.{6}[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if (length($0) == 3851) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if (length($0) == prof_n_chars || length($0) == prof_n_chars_p1) {l++;bloc=bloc"\n"$0;next} else e=1}
l==8 {if ($0 ~ /^[\003][0-9a-z]{4}[\004].$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}' "$file_"
;;
"P7")
awk --posix -v cl31_cor="$cl31_cor" '\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
NR<3 {print $0 >> cl31_cor;next}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}.$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^[\001]CL020123[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == 7501) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if ($0 ~ /^[\003][0-9a-z]{4}[\004].$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}' "$file_"
;;
"ROISSY_5M_30S")
awk --posix -v cl31_cor="$cl31_cor" '\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
NR<3 {print $0 >> cl31_cor;next}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1; printf "%s\n", $0}
l==2 {if ($0 ~ /^[\001]CL220123[\002]$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == 7500) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if ($0 ~ /^[\003][0-9a-z]{4}[\004]$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}' "$file_"
;;
"ROISSY_10M_30S")
awk --posix -v cl31_cor="$cl31_cor" '\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
NR<3 {print $0 >> cl31_cor;next}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1; printf "%s\n", $0}
l==2 {if ($0 ~ /^[\001]CL220521[\002]$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == 3850) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if ($0 ~ /^[\003][0-9a-z]{4}[\004]$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}' "$file_"
;;
esac
done
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment