Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
3e82b125
Commit
3e82b125
authored
Jan 21, 2020
by
JOSSOUD Olivier
Browse files
SSH DMC
parent
04c5d269
Pipeline
#56898
passed with stages
in 3 minutes and 17 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
107 additions
and
1 deletion
+107
-1
wimcollect/requirements_pkg.txt
wimcollect/requirements_pkg.txt
+2
-1
wimcollect/sshdmc.py
wimcollect/sshdmc.py
+105
-0
No files found.
wimcollect/requirements_pkg.txt
View file @
3e82b125
configobj==5.0.*
\ No newline at end of file
configobj==5.0.*
paramiko==2.7.*
\ No newline at end of file
wimcollect/sshdmc.py
0 → 100644
View file @
3e82b125
import
os
import
re
import
configobj
import
datetime
import
zipfile
import
paramiko
import
wimcollect.logger
as
logger
class
Collector
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
# Logger
self
.
logger
=
log
self
.
object_id
=
"SSHDMC"
# Config
self
.
config
=
config_parser
self
.
base_dir
=
self
.
config
[
self
.
object_id
][
"root_dir"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
"""Download Picarro data file from SSH server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DMC Picarro data."
)
sftp_client
=
self
.
__sftp_connect__
()
# Build source file path
picarro_id
=
self
.
config
[
self
.
object_id
][
"picarro_id"
]
picarro_number
=
re
.
sub
(
"[^0-9]"
,
""
,
picarro_id
)
source_filepath
=
self
.
base_dir
+
"/"
+
picarro_number
\
+
"/DMC_"
+
picarro_id
+
"_"
+
day
.
strftime
(
"%Y%m%d"
)
+
".zip"
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_filepath
)
success
=
self
.
__sftp_download__
(
sftp_client
,
source_filepath
,
dest_filepath
)
if
success
:
self
.
__recompress_file__
(
day
,
dest_filepath
)
sftp_client
.
close
()
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
,
source_filepath
:
str
)
->
str
:
# Build destination directory
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
"DMC"
,
"picarro"
,
self
.
config
[
self
.
object_id
][
"picarro_id"
],
str
(
day
.
year
))
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
# Build destination filename
dest_filename
=
os
.
path
.
basename
(
source_filepath
)
# Build full file path
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
dest_filename
)
return
dest_filepath
def
__recompress_file__
(
self
,
day
:
datetime
.
date
,
zip_filepath
:
str
):
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Re-compressing from ZIP to LZMA..."
)
# Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
# LZMA file will be created.
current_directory
=
os
.
path
.
dirname
(
zip_filepath
)
# Extract zip file and delete it
source_zip
=
zipfile
.
ZipFile
(
zip_filepath
,
'r'
)
files_in_zip
=
[
os
.
path
.
join
(
current_directory
,
filename
)
for
filename
in
source_zip
.
namelist
()]
source_zip
.
extractall
(
current_directory
)
source_zip
.
close
()
os
.
remove
(
zip_filepath
)
# Compress the files
compressed_filepath
=
os
.
path
.
splitext
(
zip_filepath
)[
0
]
+
".lzma"
zipf
=
zipfile
.
ZipFile
(
compressed_filepath
,
'w'
,
zipfile
.
ZIP_LZMA
)
for
filepath
in
files_in_zip
:
zipf
.
write
(
filepath
,
arcname
=
os
.
path
.
basename
(
filepath
))
zipf
.
close
()
if
os
.
path
.
exists
(
compressed_filepath
):
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Done. Archive file: "
+
compressed_filepath
)
for
file
in
files_in_zip
:
os
.
remove
(
file
)
def
__sftp_connect__
(
self
)
->
paramiko
.
SFTPClient
:
client
=
paramiko
.
SSHClient
()
client
.
set_missing_host_key_policy
(
paramiko
.
AutoAddPolicy
())
try
:
client
.
connect
(
hostname
=
self
.
config
[
self
.
object_id
][
"host"
],
username
=
self
.
config
[
self
.
object_id
][
"user"
],
key_filename
=
self
.
config
[
self
.
object_id
][
"private_key_path"
])
except
Exception
as
e
:
self
.
logger
.
write
(
self
.
object_id
,
"Failed to connect to server:
\n
"
+
str
(
e
))
raise
e
else
:
return
client
.
open_sftp
()
def
__sftp_download__
(
self
,
sftp_client
:
paramiko
.
SFTPClient
,
source_filepath
:
str
,
dest_filepath
:
str
)
->
bool
:
sftp_client
.
get
(
source_filepath
,
dest_filepath
)
success
=
os
.
path
.
exists
(
dest_filepath
)
return
success
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment