merge master

.gitlab-ci.yml

 stages:
  - install
  - test
+ - build_containers
+ - zenodo
+ - deploy
 
 .junit_template: &junit_definition
     artifacts:
@@ -35,3 +38,49 @@ test_py37:
         --cov-report=xml
         --cov-report=term
     <<: *junit_definition
+
+build_docker:
+    stage: build_containers
+    image: docker:19.03.12
+    services:
+        - docker:19.03.12-dind
+    before_script:
+        - cat /etc/os-release  # "Alpine Linux v3.12"
+        - apk add git
+        - export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'`
+        - echo $LAST_RELEASE
+    script:
+        - cd Docker
+        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+        - docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE .
+        - docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE
+    only:
+        - tags
+
+deploy_zenodo:
+  stage: zenodo
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  before_script:
+    - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - eossr-upload-new-deposit --token $ZENODO_TOKEN --sandbox False --input-dir ./build
+#    - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
+  only:
+    - tags
+
+pages:
+    stage: deploy
+    image: python:3.7-alpine
+    script:
+      - pip install -U sphinx
+      - sphinx-build -b html docs public
+    artifacts:
+      paths:
+        - public
+    only:
+      - master

Docker/Dockerfile

+FROM continuumio/miniconda3
+
+ENV PATH=/opt/conda/bin/:${PATH}
+SHELL [ "/bin/bash", "-c"]
+
+RUN conda install git pip && \
+    pip --no-cache-dir install git+https://gitlab.in2p3.fr/escape2020/wp3/eossr

README.md

@@ -7,4 +7,7 @@ https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/commits/master)
 https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/commits/master)
 
 
-ESCAPE OSSR library
+ESCAPE OSSR library
\ No newline at end of file
+
+* Code: https://gitlab.in2p3.fr/escape2020/wp3/eossr
+* Documentation: https://escape2020.pages.in2p3.fr/wp3/eossr/ 
\ No newline at end of file

codemeta.json

@@ -38,7 +38,7 @@
       "softwareVersion": ">=5.4.2"
     }
   ],
-  "keywords": [],
+  "keywords": ["jupyter-notebook"],
   "runtimePlatform": "",
   "downloadUrl": "",
   "installUrl": "",

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

+README
+======
+
+.. literalinclude:: ../README.md
+   :lines: 9-
\ No newline at end of file

docs/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+import shutil
+sys.path.insert(0, os.path.abspath('..'))
+notebook_dir = '../examples/notebooks/'
+if not os.path.exists('examples'):
+    os.mkdir('examples')
+[shutil.copy(notebook_dir + file, 'examples') for file in os.listdir(notebook_dir) if file.endswith('.ipynb')]
+
+# -- Project information -----------------------------------------------------
+
+project = 'eossr'
+copyright = '2021, ESCAPE WP3-OSSR developers and contributors'
+author = 'Thomas Vuillaume & Enrique Garcia'
+
+release = ''
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# Parsers
+source_parsers = {
+   '.md': 'recommonmark.parser.CommonMarkParser',
+}
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = ['.rst', '.md']
+
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
\ No newline at end of file

docs/docstring.rst

+=====================
+Package Documentation
+=====================
+
+Autogenerated documentation from docstrings.
+
+API module
+-------------
+
+.. automodule:: eossr.api.zenodo
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+Metadata module
+----------------------
+
+.. automodule:: eossr.metadata.codemeta2zenodo
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/examples.rst

+Examples
+========
+
+Work in progress.
+
+.. toctree::
+   :maxdepth: 0
+
+    examples/Harvest_zenodo.ipynb
+    ../examples/notebooks/Harvest_zenodo.ipynb

docs/index.rst

+.. eossr documentation master file, created by
+   sphinx-quickstart on Wed Sep  8 16:24:23 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to eossr's documentation!
+=================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   README
+   license
+   docstring
+   examples
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

docs/license.rst

+License
+=============
+
+.. include:: ../LICENSE
\ No newline at end of file

docs/make.bat

+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

eossr/api/__init__.py

 import requests
+from . import zenodo
 from .zenodo import Record, get_zenodo_records, zenodo_api_url
 
 __all__ = [
-    'Record',
+    'zenodo',
     'get_ossr_records'
 ]
 
@@ -42,3 +43,4 @@ def get_ossr_records(search='', **kwargs):
     kwargs['communities'] = escape_community
 
     return get_zenodo_records(search, **kwargs)
+

eossr/api/zenodo.py

@@ -313,7 +313,8 @@ class ZenodoAPI:
             parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
                                                           self.path_zenodo_metadata_file)
         else:
-            pass
+            print("\n ! NO codemeta.json file found. \n"
+                  "     Please add one to the ROOT directory of your project to ble able to perform the conversion.")
 
     def check_upload_to_zenodo(self):
         """
@@ -341,8 +342,7 @@ class ZenodoAPI:
             if self.exist_codemeta_file:
                 self.conversion_codemeta2zenodo()
             else:
-                print("\n ! NO codemeta.json NOR .zenodo.json file found. "
-                      "Please add one to the ROOT directory of your project.")
+                # No codemeta.json nor .zenodo.json
                 sys.exit(-1)
 
         print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")

eossr/metadata/__init__.py

+from . import codemeta2zenodo

eossr/metadata/codemeta2zenodo.py

@@ -50,7 +50,7 @@ def parse_person_schema_property(person_property, contributor_field):
     # Parse correctly the contributors
     if contributor_field in codemeta_contributors_fields:
 
-        if contributor_field == 'provider' or contributor_field == 'publisher':
+        if contributor_field in ['provider', 'publisher']:
             zenodo_person['type'] = 'Other'
         else:
             # First letter of contributor type MUST be capitalized
@@ -92,7 +92,7 @@ def add_author_metadata(zenodo_file, codemt_person_entry, person_field):
         full_contacts[zenodo_person['name']] = zenodo_person
 
     # then save each person by field and avoid duplicates
-    for person in full_contacts:
+    for person, value in full_contacts.items():
 
         if person_field in codemeta_creators_fields:
 
@@ -100,11 +100,8 @@ def add_author_metadata(zenodo_file, codemt_person_entry, person_field):
             if 'creators' not in zenodo_file:
                 zenodo_file['creators'] = []
 
-            if full_contacts[person] not in zenodo_file['creators']:
+            if value not in zenodo_file['creators']:
                 zenodo_file['creators'].append(full_contacts[person])
-            else:
-                pass  # avoid duplicates
-
         elif person_field in codemeta_contributors_fields:
 
             if 'contributors' not in zenodo_file:
@@ -112,8 +109,6 @@ def add_author_metadata(zenodo_file, codemt_person_entry, person_field):
 
             if full_contacts[person] not in zenodo_file['contributors']:
                 zenodo_file['contributors'].append(full_contacts[person])
-            else:
-                pass  # avoid duplicates
 
 
 def find_matching_metadata(codemeta_json):
@@ -171,11 +166,8 @@ def find_matching_metadata(codemeta_json):
 
     if 'softwareVersion' in codemeta_json and 'version' not in codemeta_json:
         metadata_zenodo['version'] = codemeta_json['softwareVersion']
-    elif 'version' in codemeta_json and 'softwareVersion' not in codemeta_json:
-        metadata_zenodo['version'] = codemeta_json['version']
     else:
         metadata_zenodo['version'] = codemeta_json['version']
-
     if 'keywords' in codemeta_json:
         if type(codemeta_json['keywords']) == list:
             metadata_zenodo['keywords'] = codemeta_json['keywords']

eossr/metadata/schema/.zenodo.json

+{
+  "title": "For details check https://developers.zenodo.org/#representation.",
+  "upload_type": "software",
+  "access_right": "open",
+  "publication_date": "YYYY-MM-DD",
+  "communities": [{"identifier": "escape2020"}],
+  "grants": [{"id": "10.13039/501100000780::824064"}],
+  "creators": [
+    {"name": "Name and Surname",
+      "affiliation": "Institute, Center",
+      "orcid": "0000-0000-0000-0000"}
+  ],
+  "description": "",
+  "license": "",
+  "doi": "",
+  "prereserve_doi": "",
+  "keywords": [],
+  "notes": "",
+  "related_identifiers": "",
+  "contributors": [{}],
+  "references": [],
+  "version": "",
+  "language": "",
+  "journal_title": "",
+  "journal_volume": "",
+  "journal_issue": "",
+  "journal_pages": "",
+  "conference_title": "",
+  "conference_acronym": "",
+  "conference_dates": "",
+  "conference_place": "",
+  "conference_url": "",
+  "conference_session": "",
+  "conference_session_part": "",
+  "imprint_publisher": "",
+  "imprint_isbn": "",
+  "imprint_place": "",
+  "partof_title": "",
+  "partof_pages": "",
+  "thesis_supervisors": "",
+  "thesis_university": "",
+  "subjects": "",
+  "locations": [{}],
+  "dates": [{}],
+  "method": ""
+}
\ No newline at end of file

eossr/metadata/tests/test_codemeta2zenodo.py

@@ -2,6 +2,7 @@ import json
 import unittest
 import tempfile
 from os.path import dirname, realpath, join
+from eossr.metadata import codemeta2zenodo
 
 SAMPLES_DIR = join(dirname(realpath(__file__)), "samples")
 ROOT_DIR = dirname(realpath("codemeta.json"))
@@ -60,9 +61,8 @@ zenodo_entries = [
 
 
 def test_Codemeta2ZenodoController():
-    from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController
 
-    converter = CodeMeta2ZenodoController()
+    converter = codemeta2zenodo.CodeMeta2ZenodoController()
     assert converter.codemeta_data == {}
     assert converter.zenodo_data == {}
 
@@ -84,41 +84,37 @@ def test_Codemeta2ZenodoController():
 
 
 def test_add_author_metadata():
-    from eossr.metadata.codemeta2zenodo import add_author_metadata, codemeta_allowed_person_fields
 
     with open(join(SAMPLES_DIR, "codemeta_contributors_sample.json")) as f:
         codemeta_metadata = json.load(f)
     zenodo_metadata = {}
 
-    assert all(person in codemeta_metadata.keys() for person in codemeta_allowed_person_fields)
+    assert all(person in codemeta_metadata.keys() for person in codemeta2zenodo.codemeta_allowed_person_fields)
 
-    for person in codemeta_allowed_person_fields:
-        add_author_metadata(zenodo_metadata,
-                            codemeta_metadata[person],
-                            person
-                            )
+    for person in codemeta2zenodo.codemeta_allowed_person_fields:
+        codemeta2zenodo.add_author_metadata(zenodo_metadata,
+                                            codemeta_metadata[person],
+                                            person
+                                            )
 
-    assert 'creators' in zenodo_metadata.keys()
+    assert 'creators' in zenodo_metadata
     # 4 'creators' one repeated, should not be duplicated.
     # Maintainer and Contributor. Author and Creator are the same person
     assert len(zenodo_metadata['creators']) == 3
 
-    assert 'contributors' in zenodo_metadata.keys()
+    assert 'contributors' in zenodo_metadata
     # Editor, Producer, Publisher, Provider and Sponsor
     assert len(zenodo_metadata['contributors']) == 5
 
 
 def test_parse_person_schema_property():
-    from eossr.metadata.codemeta2zenodo import \
-        parse_person_schema_property, \
-        codemeta_contributors_fields
 
     with open(join(SAMPLES_DIR, "codemeta_contributors_sample.json")) as f:
         codemeta_metadata = json.load(f)
 
-    for person in codemeta_contributors_fields:
-        zenodo_metadata = parse_person_schema_property(codemeta_metadata[person],
-                                                       person)
+    for person in codemeta2zenodo.codemeta_contributors_fields:
+        zenodo_metadata = codemeta2zenodo.parse_person_schema_property(codemeta_metadata[person],
+                                                                       person)
         if person == 'editor':
             assert zenodo_metadata['type'] == 'Editor'
         elif person == 'producer':
@@ -131,20 +127,18 @@ def test_parse_person_schema_property():
 
 class TestConverting(unittest.TestCase):
     def test_sample_file_conversion(self):
-        from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
 
         outfile = tempfile.NamedTemporaryFile(delete=True)
-        parse_codemeta_and_write_zenodo_metadata_file(
+        codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file(
             join(SAMPLES_DIR, "codemeta_sample1.json"), outfile.name
         )
 
         json.load(outfile)
 
     def test_root_codemeta_conversion(self):
-        from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
 
         outfile = tempfile.NamedTemporaryFile(delete=True)
-        parse_codemeta_and_write_zenodo_metadata_file(
+        codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file(
             join(ROOT_DIR, "codemeta.json"), outfile.name
         )
         json.load(outfile)
@@ -152,9 +146,8 @@ class TestConverting(unittest.TestCase):
 
 class TestLicense(unittest.TestCase):
     def test_license1(self):
-        from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController
 
-        converter = CodeMeta2ZenodoController()
+        converter = codemeta2zenodo.CodeMeta2ZenodoController()
         converter.convert_license()
         assert 'license' not in converter.zenodo_data