Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
IPSL
SIRTA
CL31
CL raw to 1a
Commits
aecb9f54
Commit
aecb9f54
authored
Jan 20, 2022
by
Marc-Antoine Drouin
Browse files
update script to check raw data format
parent
192907ea
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
59 additions
and
4 deletions
+59
-4
run_raw2l1.sh
run_raw2l1.sh
+59
-4
No files found.
run_raw2l1.sh
View file @
aecb9f54
...
...
@@ -109,7 +109,7 @@ n=0
# Recupere les parametres de la lignes de commande
#------------------------------------------------------------------------------
UNZIP
=
0
while
getopts
:d:e:s:t:n:c:uhv option
;
do
while
getopts
:d:e:s:t:n:c:
a:
uhv option
;
do
case
$option
in
d
)
date
=
"
$OPTARG
"
;;
h
)
...
...
@@ -129,6 +129,7 @@ while getopts :d:e:s:t:n:c:uhv option; do
n
)
DEBUG
=
${
OPTARG
}
;;
c
)
fich_conf
=
"
$OPTARG
"
;;
u
)
UNZIP
=
1
;;
a
)
fich_type
=
"
$OPTARG
"
;;
# SIRTA ou MF (format change legerement selon le logiciel d'acquisition)
# permet de sauter des options inconnues sans sortir de while
?
)
while
[[
$OPTIND
-le
$#
&&
${
!OPTIND
:0:1
}
!=
'-'
]]
;
do
OPTIND
=
$((
OPTIND
+
1
))
;
done
;;
*
)
ERREUR
=
1
;;
...
...
@@ -151,6 +152,11 @@ if [[ $ERREUR != 0 ]]; then
exit
1
fi
if
[[
$fich_type
!=
"SIRTA"
&&
$fich_type
!=
"MF"
]]
;
then
echo
"Le type de fichier doit etre SIRTA ou MF"
exit
1
fi
let
n--
while
[[
$n
-ge
0
]]
;
do
if
[[
-z
${
liste_entree
[n]
}
]]
;
then
...
...
@@ -205,6 +211,7 @@ if [[ ${#fich_entree[*]} -lt 2 ]]; then
fi
# if unzipping raw files is needed
#------------------------------------------------------------------------------
if
[[
$UNZIP
==
0
]]
;
then
input_files
=(
"
${
fich_entree
[@]
}
"
)
elif
[[
$UNZIP
==
1
]]
;
then
...
...
@@ -220,12 +227,58 @@ elif [[ $UNZIP == 1 ]]; then
mapfile
-t
input_files < <
(
find
"
$tmp_data
"
-name
"*.dat"
|
sort
)
fi
# check format of data message
#------------------------------------------------------------------------------
# create temp directory for data
tmp_check
=
$(
mktemp
-d
-p
"
$REP_TRAVAIL
"
)
for
file_
in
"
${
input_files
[@]
}
"
;
do
cl31_cor
=
"
$tmp_check
/
${
file_
%.*
}
.cor"
if
[[
$fich_type
==
"SIRTA"
]]
;
then
awk
--posix
-v
cl31_cor
=
"
$cl31_cor
"
'\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
NR<3 {print $0 >> cl31_cor;next}
l==1 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}.$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^[\001]CL020521[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (length($0) == 3851) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if ($0 ~ /^[\003][0-9a-z]{4}[\004].$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}'
"
$file_
"
elif
[[
$fich_type
==
"MF"
]]
;
then
awk
--posix
-v
cl31_cor
=
"
$cl31_cor
"
'\
BEGIN {l=1;p=0;e=0;printf "" > cl31_cor}
l==1 {if ($0 ~ /^[\001]$/) {l++;bloc=$0;e=0;next} else if (e==1) {next} else e=1}
l==2 {if ($0 ~ /^-[0-9-]{10} [0-9:]{8}$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==3 {if ($0 ~ /^ CL120521[\002].$/) {l++;bloc=bloc"\n"$0;next} else e=1}
l==4 {if (NF==5) {l++;bloc=bloc"\n"$0;next} else e=1}
l==5 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==6 {if (NF==10) {l++;bloc=bloc"\n"$0;next} else e=1}
l==7 {if (length($0) == 3851) {l++;bloc=bloc"\n"$0;next} else e=1}
l==8 {if ($0 ~ /^[\003][0-9a-z]{4}[\004].$/) {p=1;bloc=bloc"\n"$0} else e=1}
p==1 {printf bloc"\n" >> cl31_cor;l=1;p=0}
e==1 {err++;print NR;l=1}
END {if ($0 ~ "File") printf "\n%s\n",$0>>cl31_cor; if (err>0)print "err=",err}'
"
$file_
"
fi
done
mapfile
-t
input_files < <
(
find
"
$tmp_check
"
-name
"*.cor"
|
sort
)
# Recuperation des fichiers de sortie
#------------------------------------------------------------------------------
fich_sortie
=
$(
cat
$liste_sortie
)
fich_sortie
=
$(
cat
"
$liste_sortie
"
)
# fichier temporaire
tmpfile
=
$(
mktemp
$REP_TRAVAIL
/test_XXXXX.nc
)
tmpfile
=
$(
mktemp
"
$REP_TRAVAIL
"
/test_XXXXX.nc
)
# conversion avec raw2l1
#------------------------------------------------------------------------------
...
...
@@ -244,7 +297,7 @@ if [[ $? == 0 ]]; then
consignerMsg
"suppression des données n'appartenant au jour traite"
eval
$cmd
else
\r
m
$tmpfile
\r
m
"
$tmpfile
"
if
[[
$UNZIP
==
1
]]
;
then
\r
m
-rf
"
$tmp_data
"
...
...
@@ -259,6 +312,8 @@ if [[ $UNZIP == 1 ]]; then
\r
m
-rf
"
$tmp_data
"
fi
\r
m
-rf
"
$tmp_check
"
if
((
$?
>
1
))
;
then
exit
1
fi
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment