Marc-Antoine Drouin · Marc-Antoine Drouin · Marc-Antoine Drouin · Marc-Antoine Drouin · Marc-Antoine Drouin · Marc-Antoine Drouin
20 changed files
--- a/.gitignore
+++ b/.gitignore
-sonar-project.properties
-raw2l1/doc/build/
-raw2l1.sublime-workspace
-.sonar/
-raw2l1/.coverage
-.reports/
-.vscode/
-tags
\ No newline at end of file
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vscode
+
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
-image: continuumio/anaconda3:latest
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"

 stages:
  - test

-# Pick zero or more services to be used on all builds.
-# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
-#services:
-#  - mysql:latest
-#  - postgres:latest
+cache:
+  paths:
+    - "$CI_PROJECT_DIR/pip-cache"
+  key: "$CI_PROJECT_ID"

-# This folder is cached between builds
-# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
-#cache:
-#  paths:
-#  - ~/.conda/
-
-# This is a basic example for a gem or script which doesn't use
-# services such as redis or postgres
 before_script:
  - python -V
-  - conda create -n raw2l1 python=2 
-  - source activate raw2l1
-  - conda install --file requirements.txt
+  - pip install -r requirements.txt
  - cd raw2l1

-run_test:
+test_py36:
+  stage: test
+  image: python:3.6
+  script:
+    - py.test --cov
+  coverage: '/\d+\%\s*$/'
+
+test_py37:
  stage: test
+  image: python:3.7
  script:
-    - nosetests
-    
-coverage:
-    stage: test
-    script:
-        - nosetests -q --cover-html-dir=build --cover-html
-        - coverage report -m
-    coverage: '/TOTAL.+ ([0-9]{1,3}%)/'
-    artifacts:
-        paths:
-            - build
-        expire_in: 1 day
+    - py.test --cov
+  coverage: '/\d+\%\s*$/'
+  allow_failure: true
--- a/CHANGELOG
+++ b/CHANGELOG
-* V2.1.19
+* v3.0.2
+    + hatpro reader
+        - improve syncing of brightness and meteo data
+    + WLS70 10 min reader
+        - correct bug when localization is not well formatted
+
+* v3.0.1
+    + correct a bug converting `units = 1` into an integer attribute. Units are now always interpreted as string.
+
+* v3.0.0
+    + make raw2l1 compatible with python 3.6 and 3.7
+    + drop compatibility with python2
+    + uniformize source code format using black
+    + correct bugs with readers
+    + move repository on IN2P3s gitlab
+    + add continuous integration
+
+* v2.1.19
    + SIGMASPACE miniMPL reader
        - correct bug with `laser_energy`. 273.15 was added to the variable by mistake


--- a/README.md
+++ b/README.md
+[![pipeline status](https://gitlab.in2p3.fr/ipsl/sirta/raw2l1/badges/master/pipeline.svg)](https://gitlab.in2p3.fr/ipsl/sirta/raw2l1/commits/master) [![coverage report](https://gitlab.in2p3.fr/ipsl/sirta/raw2l1/badges/master/coverage.svg)](https://gitlab.in2p3.fr/ipsl/sirta/raw2l1/commits/master)
+
 # raw2l1

 Code to convert raw LIDAR data into normalized netCDF files
@@ -16,7 +18,7 @@ see requirements.txt file

 the repository contains some example files allowing you to test the code

- go to raw2l1 directory 
+- go to raw2l1 directory
 - you can get the list of input arguments using the command:

 ```
@@ -24,7 +26,7 @@ python raw2l1.py -h
 ```


- to convert a LUFFT CHM15k file 
+- to convert a LUFFT CHM15k file

 ```
 python '20150427' raw2l1 conf/conf_lufft_chm15k-nimbus_toprof_netcdf4.ini test/input/Jenoptik_chm15k/20150427_SIRTA_CHM150101_000.nc test/output/test_lufft_sirta.nc

--- a/raw2l1/.gitignore
+++ b/raw2l1/.gitignore
-*.pyc
-.project
-.pydevproject
--- a/raw2l1/conf/conf_campbell_cs135_eprofile.ini
+++ b/raw2l1/conf/conf_campbell_cs135_eprofile.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -28,6 +28,7 @@ netcdf4_compression_level = 1
 timestamp_fmt = New record %d.%m.%Y %H:%M:%S
 missing_float = -999.
 missing_int = -9
+file_encoding = utf8

 ; Variables required by TOPROF chosen format
 ;==============================================================================

--- a/raw2l1/conf/conf_campbell_cs135_toprof.ini
+++ b/raw2l1/conf/conf_campbell_cs135_toprof.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -26,6 +26,7 @@ netcdf_format = NETCDF3_CLASSIC
 timestamp_fmt = New record %d.%m.%Y %H:%M:%S
 missing_float = -999.
 missing_int = -9
+file_encoding = utf8

 ; Variables required by TOPROF chosen format
 ;==============================================================================
@@ -35,7 +36,7 @@ missing_int = -9
 [global]
 site_location = Palaiseau, France
 instrument_id = $reader_data$, instrument_id
-sofware_id = $reader_data$, software_version
+sofware_id = $reader_data$, os
 overlap_is_corrected = true
 overlap_function = false
 instrument_version_number = instrument_version_number

--- a/raw2l1/conf/conf_campbell_cs135_toprof_netcdf4.ini
+++ b/raw2l1/conf/conf_campbell_cs135_toprof_netcdf4.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -28,6 +28,7 @@ netcdf4_compression_level = 1
 timestamp_fmt = New record %d.%m.%Y %H:%M:%S
 missing_float = -999.
 missing_int = -9
+file_encoding = utf8

 ; Variables required by TOPROF chosen format
 ;==============================================================================
@@ -37,7 +38,7 @@ missing_int = -9
 [global]
 site_location = Palaiseau, France
 instrument_id = $reader_data$, instrument_id
-sofware_id = $reader_data$, software_version
+sofware_id = $reader_data$, os
 overlap_is_corrected = true
 overlap_function = false
 instrument_version_number = instrument_version_number

--- a/raw2l1/conf/conf_jenoptik_chm15k.ini
+++ b/raw2l1/conf/conf_jenoptik_chm15k.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;------------------------------------------------------------------------------

 ; General configuration of the processing

--- a/raw2l1/conf/conf_leosphere_wls70_10min.ini
+++ b/raw2l1/conf/conf_leosphere_wls70_10min.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;------------------------------------------------------------------------------

 ; General configuration of the processing
@@ -22,6 +22,7 @@ netcdf_format = NETCDF4
 [reader_conf]
 missing_float = -999.
 missing_int = -9
+file_encoding = "ISO-8859-1"

 ; Global attribute of the netCDF file
 ;------------------------------------------------------------------------------

--- a/raw2l1/conf/conf_leosphere_wls70_10s.ini
+++ b/raw2l1/conf/conf_leosphere_wls70_10s.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;------------------------------------------------------------------------------

 ; General configuration of the processing
@@ -22,6 +22,7 @@ netcdf_format = NETCDF4
 [reader_conf]
 missing_float = -999.
 missing_int = -9
+file_encoding = "ISO-8859-1"

 ; Global attribute of the netCDF file
 ;------------------------------------------------------------------------------

--- a/raw2l1/conf/conf_leosphere_wls7_10min.ini
+++ b/raw2l1/conf/conf_leosphere_wls7_10min.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;------------------------------------------------------------------------------

 ; General configuration of the processing
@@ -22,6 +22,7 @@ netcdf_format = NETCDF4
 [reader_conf]
 missing_float = -999.
 missing_int = -9
+file_encoding = "ISO-8859-1"

 ; Global attribute of the netCDF file
 ;------------------------------------------------------------------------------

--- a/raw2l1/conf/conf_leosphere_wls7_1s.ini
+++ b/raw2l1/conf/conf_leosphere_wls7_1s.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;------------------------------------------------------------------------------

 ; General configuration of the processing
@@ -22,6 +22,7 @@ netcdf_format = NETCDF4
 [reader_conf]
 missing_float = -999.
 missing_int = -9
+file_encoding = "ISO-8859-1"

 ; Global attribute of the netCDF file
 ;------------------------------------------------------------------------------

--- a/raw2l1/conf/conf_lufft_chm15k-nimbus-uk-metoffice_eprofile.ini
+++ b/raw2l1/conf/conf_lufft_chm15k-nimbus-uk-metoffice_eprofile.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -182,15 +182,6 @@ missing_value = -999.
 _FillValue = -999.
 value = -999.

-[tilt_angle]
-type = $float$
-dim = $none$
-long_name = telescope 0 tilt angle wrt nadir
-units = degree
-missing_value = -999.
-_FillValue = -999.
-value = -999.
-
 [temperature_optical_module]
 dim = time
 type = $integer$

--- a/raw2l1/conf/conf_lufft_chm15k-nimbus_eprofile.ini
+++ b/raw2l1/conf/conf_lufft_chm15k-nimbus_eprofile.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -25,7 +25,7 @@ netcdf4_compression_level = 1
 ; Special option for the reader (optional)
 ;------------------------------------------------------------------------------
 [reader_conf]
-overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg ; optional
+overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg
 missing_float = -999.
 missing_int = -9

@@ -184,15 +184,6 @@ missing_value = -999.
 _FillValue = -999.
 value = -999.

-[tilt_angle]
-type = $float$
-dim = $none$
-long_name = telescope 0 tilt angle wrt nadir
-units = degree
-missing_value = -999.
-_FillValue = -999.
-value = -999.
-
 [temperature_optical_module]
 dim = time
 type = $integer$

--- a/raw2l1/conf/conf_lufft_chm15k-nimbus_toprof.ini
+++ b/raw2l1/conf/conf_lufft_chm15k-nimbus_toprof.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -23,7 +23,7 @@ netcdf_format = NETCDF3_CLASSIC
 ; Special option for the reader (optional)
 ;------------------------------------------------------------------------------
 [reader_conf]
-overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg ; optional
+overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg

 ; Variables required by TOPROF chosen format
 ;==============================================================================

--- a/raw2l1/conf/conf_lufft_chm15k-nimbus_toprof_netcdf4.ini
+++ b/raw2l1/conf/conf_lufft_chm15k-nimbus_toprof_netcdf4.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -25,7 +25,7 @@ netcdf4_compression_level = 1
 ; Special option for the reader (optional)
 ;------------------------------------------------------------------------------
 [reader_conf]
-overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg ; optional
+overlap_file = test/input/jenoptik_chm15k/TUB140013_20150211_4096.cfg
 missing_float = -999.
 missing_int = -9

@@ -56,7 +56,7 @@ Conventions = CF-1.6
 ;------------------------------------------------------------------------------
 [time]
 dim = time
-type = $double$
+type = $time$
 standard_name = time
 units = days since 1970-01-01 00:00:00
 calendar = standard

--- a/raw2l1/conf/conf_rpg_hatpro_bl00-l1-tb_toprof_netcdf4.ini
+++ b/raw2l1/conf/conf_rpg_hatpro_bl00-l1-tb_toprof_netcdf4.ini
@@ -7,7 +7,7 @@
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
 ; version: 2.1.2
-; SIRTA IPSL/CNRS/EP 2014-2016
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;

--- a/raw2l1/conf/conf_rpg_hatpro_bl00-l2-ta_toprof_netcdf4.ini
+++ b/raw2l1/conf/conf_rpg_hatpro_bl00-l2-ta_toprof_netcdf4.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;

--- a/raw2l1/conf/conf_rpg_hatpro_l1-tb_toprof_netcdf4.ini
+++ b/raw2l1/conf/conf_rpg_hatpro_l1-tb_toprof_netcdf4.ini
@@ -6,8 +6,8 @@
 ; | v / /\ || 'V' | / /| |_`7 |
 ; |_|_\_||_|!_/ \_!|___|___||_|
 ;
-; version: 2.1.19
-; SIRTA IPSL/CNRS/EP 2014-2016
+; version: 3.0.2
+; SIRTA IPSL/CNRS/EP 2014-2019
 ;
 ; for TOPROF netCDF-CF format
 ;
@@ -192,7 +192,7 @@ _FillValue = -999.
 comments = azimuth angle same as microwave radiometer
 valid_min = 2.7
 valid_max = 330.
-value = tb_irp
+value = $reader_data$, tb_irp

 [ele_irp]
 dim = time