Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
0b4040ac
Commit
0b4040ac
authored
Jan 22, 2020
by
JOSSOUD Olivier
Browse files
Create Collector superclass to manage common log and config_parser.
parent
662aeb9d
Pipeline
#57022
passed with stages
in 2 minutes and 33 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
52 additions
and
35 deletions
+52
-35
wimcollect/ftp.py
wimcollect/ftp.py
+4
-8
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+2
-2
wimcollect/ftpopar.py
wimcollect/ftpopar.py
+2
-2
wimcollect/httpddu.py
wimcollect/httpddu.py
+3
-8
wimcollect/main.py
wimcollect/main.py
+3
-3
wimcollect/sshdmc.py
wimcollect/sshdmc.py
+9
-9
wimcollect/utils.py
wimcollect/utils.py
+29
-3
No files found.
wimcollect/ftp.py
View file @
0b4040ac
...
...
@@ -3,17 +3,13 @@ import ftplib
import
codecs
import
wimcollect.logger
as
logger
import
wimcollect.utils
as
utils
class
FtpCollector
:
class
FtpCollector
(
utils
.
Collector
)
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
,
object_id
:
str
):
# Logger
self
.
logger
=
log
self
.
object_id
=
object_id
# Config
self
.
config
=
config_parser
def
__init__
(
self
,
object_id
:
str
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
utils
.
Collector
.
__init__
(
self
,
object_id
,
config_parser
,
log
)
self
.
base_dir
=
self
.
config
[
self
.
object_id
][
"root_dir"
]
def
_ftp_connect_
(
self
)
->
ftplib
.
FTP
:
...
...
wimcollect/ftpcea.py
View file @
0b4040ac
...
...
@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class
Collector
(
ftp
.
FtpCollector
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
self
.
object_id
=
"FTPCEA"
ftp
.
FtpCollector
.
__init__
(
self
,
config_parser
,
log
,
self
.
object_id
)
ftp
.
FtpCollector
.
__init__
(
self
,
self
.
object_id
,
config_parser
,
log
)
####################################################################################################################
# Picarro
...
...
wimcollect/ftpopar.py
View file @
0b4040ac
...
...
@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class
Collector
(
ftp
.
FtpCollector
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
self
.
object_id
=
"FTPOPAR"
ftp
.
FtpCollector
.
__init__
(
self
,
config_parser
,
log
,
self
.
object_id
)
ftp
.
FtpCollector
.
__init__
(
self
,
self
.
object_id
,
config_parser
,
log
)
def
download_maido_ftir
(
self
,
first_day
:
datetime
.
date
,
last_day
:
datetime
.
date
):
"""Download all Picarro files from FTP server.
...
...
wimcollect/httpddu.py
View file @
0b4040ac
...
...
@@ -7,15 +7,10 @@ import wimcollect.utils as utils
import
wimcollect.logger
as
logger
class
Collector
:
class
Collector
(
utils
.
Collector
)
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
# Logger
self
.
logger
=
log
self
.
object_id
=
"HTTPDDU"
# Config
self
.
config
=
config_parser
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
utils
.
Collector
.
__init__
(
self
,
"HTTPDDU"
,
config_parser
,
log
)
self
.
base_url
=
self
.
config
[
self
.
object_id
][
"base_url"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
...
...
wimcollect/main.py
View file @
0b4040ac
...
...
@@ -30,13 +30,13 @@ def main():
yesterday
=
datetime
.
datetime
.
now
()
-
datetime
.
timedelta
(
1
)
httpddu_col
=
httpddu
.
Collector
(
config_parser
,
log
)
httpddu_col
=
httpddu
.
Collector
()
httpddu_col
.
download_picarro
(
yesterday
)
sshdmc_col
=
sshdmc
.
Collector
(
config_parser
,
log
)
sshdmc_col
=
sshdmc
.
Collector
()
sshdmc_col
.
download_picarro
(
yesterday
)
vizual
=
quickviz
.
Visualizator
(
config_parser
,
log
)
vizual
=
quickviz
.
Visualizator
()
vizual
.
create_data_source_file
(
"DDU"
,
"HIDS2189"
,
yesterday
)
vizual
.
create_data_source_file
(
"DMC"
,
"HIDS2319"
,
yesterday
)
...
...
wimcollect/sshdmc.py
View file @
0b4040ac
...
...
@@ -8,15 +8,10 @@ import wimcollect.utils as utils
import
wimcollect.logger
as
logger
class
Collector
:
class
Collector
(
utils
.
Collector
)
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
# Logger
self
.
logger
=
log
self
.
object_id
=
"SSHDMC"
# Config
self
.
config
=
config_parser
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
utils
.
Collector
.
__init__
(
self
,
"SSHDMC"
,
config_parser
,
log
)
self
.
base_dir
=
self
.
config
[
self
.
object_id
][
"root_dir"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
...
...
@@ -79,6 +74,11 @@ class Collector:
return
client
.
open_sftp
()
def
__sftp_download__
(
self
,
sftp_client
:
paramiko
.
SFTPClient
,
source_filepath
:
str
,
dest_filepath
:
str
)
->
bool
:
sftp_client
.
get
(
source_filepath
,
dest_filepath
)
try
:
sftp_client
.
get
(
source_filepath
,
dest_filepath
)
except
FileNotFoundError
:
msg
=
"File not found: "
+
source_filepath
self
.
logger
.
write
(
self
.
object_id
,
msg
)
raise
FileNotFoundError
(
msg
)
success
=
os
.
path
.
exists
(
dest_filepath
)
return
success
wimcollect/utils.py
View file @
0b4040ac
import
os
import
re
import
sys
import
configobj
import
datetime
import
pkgutil
import
zipfile
import
paramiko
import
wimcollect.logger
as
logger
class
Collector
:
def
__init__
(
self
,
object_id
:
str
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
# Config
if
config_parser
is
None
:
self
.
config
=
get_config_parser
()
else
:
self
.
config
=
config_parser
# Logger
if
log
is
None
:
self
.
logger
=
logger
.
Logger
(
self
.
config
)
else
:
self
.
logger
=
log
self
.
object_id
=
object_id
def
get_config_parser
()
->
configobj
.
ConfigObj
:
# Configuration file
pkgpath
=
os
.
path
.
dirname
(
pkgutil
.
get_loader
(
"wimcollect"
).
path
)
conf_file_path
=
os
.
path
.
join
(
pkgpath
,
"config"
,
"settings.ini"
)
if
not
os
.
path
.
exists
(
conf_file_path
):
msg
=
"Configuration file not found in ["
+
conf_file_path
+
"]"
sys
.
stderr
.
write
(
msg
)
raise
FileNotFoundError
(
msg
)
config_parser
=
configobj
.
ConfigObj
(
conf_file_path
)
return
config_parser
def
recompress_file
(
zip_filepath
:
str
):
# Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
# LZMA file will be created.
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment