Merge branch 'add_codemeta2zdenodo_tests' into 'master'

add tests for codemeta2zenodo module

See merge request !3

.gitignore

+# Compiled files
+__pycache__
+.pytest_cache
+.coverage
+
+# Packages/installer info
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+
+# Mac OSX
+.DS_Store
+default.profraw
+
+# Pycharm editor project files
+.idea
\ No newline at end of file

.gitlab-ci.yml

+stages:
+ - install
+ - test
+
+.junit_template: &junit_definition
+    artifacts:
+      reports:
+        junit: "junit*.xml"
+
+
+install_py37:
+  stage: install
+  image: python:3.7-buster
+  script:
+    - apt-get -y update
+    - python setup.py install
+    - eossr-codemeta2zenodo --help
+    - eossr-codemeta2zenodo --input_codemeta_file ./codemeta.json
+    - cat .zenodo.json
+  only:
+    - branches
+
+test_py37:
+    stage: test
+    image: python:3.7-buster
+    script:
+      - apt-get -y update
+      - pip install pytest pytest-cov
+      - pip install -e .
+      - pytest eossr/
+        --junitxml=junit_py37.xml
+        --color=yes
+        --verbose
+        --cov=eossr
+        --cov-report=xml
+        --cov-report=term
+    <<: *junit_definition

README.md

 # eossr
 
+[![pipeline status](https://gitlab.in2p3.fr/escape2020/wp3/eossr/badges/master/pipeline.svg)](
+https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/commits/master)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![coverage report](https://gitlab.in2p3.fr/escape2020/wp3/eossr/badges/master/coverage.svg)](
+https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/commits/master)
+
+
 ESCAPE OSSR library
\ No newline at end of file

codemeta.json

@@ -39,13 +39,13 @@
     }
   ],
   "keywords": [],
-  "identifier": "",
   "runtimePlatform": "",
   "downloadUrl": "",
   "installUrl": "",
   "releaseNotes": "",
   "datePublished": "",
   "dateModified": "",
+  "operatingSystem":  "",
   "maintainer": {
     "@type": "Person",
     "@id": "https://orcid.org/0000-0003-2224-4594",

eossr/metadata/tests/samples/codemeta_contributors_sample.json

+{
+  "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
+  "@type": "SoftwareSourceCode",
+  "name": "Codemeta sample with all king of accepted Zenodo schema person properties.",
+  "version": "0.1",
+  "author": [
+    {
+      "@type": "Person",
+      "givenName": "Author-and-Creator-same-person-name",
+      "familyName": "Author-and-Creator-same-person-surname"
+    }
+  ],
+  "creator": {
+      "@type": "Person",
+      "givenName": "Author-and-Creator-same-person-name",
+      "familyName": "Author-and-Creator-same-person-surname"
+  },
+  "maintainer": {
+      "@type": "Person",
+      "givenName": "Maintainer-name",
+      "familyName": "Maintainer-surname"
+    },
+  "contributor": [
+    {
+      "@type": "Person",
+      "givenName": "Contributor-name",
+      "familyName": "Contributor-surname"
+    }
+  ],
+  "producer": {
+    "@type": "Person",
+    "givenName": "Producer-name",
+    "familyName": "Producer-surname"
+  },
+  "editor": {
+    "@type": "Person",
+    "givenName": "Editor-name",
+    "familyName": "Editor-surname"
+  },
+  "publisher": {
+    "@type": "Person",
+    "givenName": "Publisher-name",
+    "familyName": "Publisher-surname"
+  },
+  "provider": {
+    "@type": "Person",
+    "givenName": "Provider-name",
+    "familyName": "Provider-surname"
+  },
+  "sponsor": {
+    "@type": "Person",
+    "givenName": "Sponsor-name",
+    "familyName": "Sponsor-surname"
+  }
+}
\ No newline at end of file

eossr/metadata/tests/samples/codemeta_sample1.json

+{
+  "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
+  "@type": "SoftwareSourceCode",
+  "identifier": "CodeMeta",
+  "description": "CodeMeta is a concept vocabulary that can be used to standardize the exchange of software metadata across repositories and organizations.",
+  "name": "CodeMeta: Minimal metadata schemas for science software and code, in JSON-LD",
+  "codeRepository": "https://github.com/codemeta/codemeta",
+  "issueTracker": "https://github.com/codemeta/codemeta/issues",
+  "license": "https://spdx.org/licenses/Apache-2.0",
+  "version": "2.0",
+  "author": [
+    {
+      "@type": "Person",
+      "givenName": "Carl",
+      "familyName": "Boettiger",
+      "email": "cboettig@gmail.com",
+      "@id": "http://orcid.org/0000-0002-1642-628X"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Matthew B.",
+      "familyName": "Jones",
+      "email": "jones@nceas.ucsb.edu",
+      "@id": "http://orcid.org/0000-0003-0077-4738"
+    }
+  ],
+  "contributor": [
+    {
+      "@type": "Person",
+      "givenName": "Abby Cabunoc",
+      "familyName": "Mayes",
+      "email": "abbycabs@gmail.com"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Arfon",
+      "familyName": "Smith",
+      "email": "arfon.smith@gmail.com",
+      "@id": "http://orcid.org/0000-0002-3957-2474"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Peter",
+      "familyName": "Slaughter",
+      "email": "slaughter@nceas.ucsb.edu",
+      "@id": "http://orcid.org/0000-0002-2192-403X"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Kyle",
+      "familyName": "Niemeyer",
+      "email": "Kyle.Niemeyer@oregonstate.edu",
+      "@id": "http://orcid.org/0000-0003-4425-7097"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Yolanda",
+      "familyName": "Gil",
+      "email": "GIL@ISI.EDU",
+      "@id": "http://orcid.org/0000-0001-8465-8341"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Krzysztof",
+      "familyName": "Nowak"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Martin",
+      "familyName": "Fenner",
+      "@id": "http://orcid.org/0000-0003-1419-2405"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Mark",
+      "familyName": "Hahnel",
+      "@id": "http://orcid.org/0000-0003-4741-0309"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Luke",
+      "familyName": "Coy",
+      "email": "luke.coy@rit.edu"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Alice",
+      "familyName": "Allen",
+      "email": "aallen@ascl.net",
+      "@id": "http://orcid.org/0000-0003-3477-2845"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Mercè",
+      "familyName": "Crosas",
+      "@id": "http://orcid.org/0000-0003-1304-1939"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Ashley",
+      "familyName": "Sands",
+      "@id": "http://orcid.org/0000-0001-5636-0433"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Neil",
+      "familyName": "Chue Hong",
+      "email": "n.chuehong@epcc.ed.ac.uk",
+      "@id": "http://orcid.org/0000-0002-8876-7606"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Patricia",
+      "familyName": "Cruse",
+      "@id": "http://orcid.org/0000-0002-9300-5278"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Dan",
+      "familyName": "Katz",
+      "email": "dskatz@illinois.edu",
+      "@id": "http://orcid.org/0000-0003-2720-0339"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Carole",
+      "familyName": "Goble",
+      "email": "carole.goble@manchester.ac.uk",
+      "@id": "http://orcid.org/0000-0003-1219-2137"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Carl",
+      "familyName": "Boettiger",
+      "email": "cboettig@gmail.com",
+      "@id": "http://orcid.org/0000-0002-1642-628X"
+    },
+    {
+      "@type": "Person",
+      "givenName": "Stephan",
+      "familyName": "Druskat",
+      "email": "mail@sdruskat.net",
+      "@id": "http://orcid.org/0000-0003-4925-7248"
+    }
+  ],
+  "maintainer": {
+    "@type": "Person",
+    "givenName": "Carl",
+    "familyName": "Boettiger",
+    "email": "cboettig@gmail.com",
+    "@id": "http://orcid.org/0000-0002-1642-628X"
+  },
+  "contIntegration": "https://travis-ci.org/codemeta/codemeta",
+  "developmentStatus": "active",
+  "downloadUrl": "https://github.com/codemeta/codemeta/archive/2.0.zip",
+  "funder": { 
+          "@id": "https://doi.org/10.13039/100000001",
+          "@type": "Organization",
+          "name": "National Science Foundation"
+  },
+  "funding":"1549758; Codemeta: A Rosetta Stone for Metadata in Scientific Software",
+  "keywords": [
+    "metadata",
+    "software"
+  ],
+  "dateCreated":"2017-06-05",
+  "datePublished":"2017-06-05",
+  "programmingLanguage": "JSON-LD"
+}

eossr/metadata/tests/test_codemeta2zenodo.py

+import json
+import unittest
+import tempfile
+from os.path import dirname, realpath, join
+
+SAMPLES_DIR = join(dirname(realpath(__file__)), "samples")
+ROOT_DIR = dirname(realpath("codemeta.json"))
+
+
+codemeta_entries = [
+    "@context",
+    "@type",
+    "name",
+    "description",
+    "license",
+    "softwareVersion",
+    "developmentStatus",
+    "codeRepository",
+    "dateCreated",
+    "isAccessibleForFree",
+    "isPartOf",
+    "contIntegration",
+    "issueTracker",
+    "readme",
+    "buildInstructions",
+    "operatingSystem",
+    "author",
+    "contributor",
+    "maintainer",
+    "funder",
+    "funding",
+    "programmingLanguage",
+    "softwareRequirements",
+    "keywords",
+    "downloadUrl",
+    "installUrl",
+    "dateModified",
+    "datePublished",
+    "runtimePlatform",
+    "releaseNotes"
+]
+
+zenodo_entries = [
+    "upload_type",
+    "title",
+    "description",
+    "language",
+    "access_right",
+    "version",
+    "keywords",
+    "notes",
+    "license",
+    "publication_date",
+    "creators",
+    # #"communities",
+    # #"grants",
+    # #"contributors",
+    # #"references",
+]
+
+
+def test_Codemeta2ZenodoController():
+    from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController
+
+    converter = CodeMeta2ZenodoController()
+    assert converter.codemeta_data == {}
+    assert converter.zenodo_data == {}
+
+    codemeta_file = join(ROOT_DIR, "codemeta.json")
+    converter.load_codemeta(codemeta_file)
+    assert converter.codemeta_data != {}
+    assert all(key in converter.codemeta_data.keys() for key in codemeta_entries)
+
+    converter.convert()
+    assert converter.zenodo_data != {}
+    assert converter.zenodo_data["language"] == "eng"
+    assert converter.zenodo_data["access_right"] == "open"
+    assert all(key in converter.zenodo_data.keys() for key in zenodo_entries)
+
+    converter.add_escape2020_community()
+    assert converter.zenodo_data["communities"] == [{"identifier": "escape2020"}]
+    converter.add_escape2020_grant()
+    assert converter.zenodo_data["grants"] == [{"id": "10.13039/501100000780::824064"}]
+
+
+def test_add_author_metadata():
+    from eossr.metadata.codemeta2zenodo import add_author_metadata, codemeta_allowed_person_fields
+
+    with open(join(SAMPLES_DIR, "codemeta_contributors_sample.json")) as f:
+        codemeta_metadata = json.load(f)
+    zenodo_metadata = {}
+
+    assert all(person in codemeta_metadata.keys() for person in codemeta_allowed_person_fields)
+
+    for person in codemeta_allowed_person_fields:
+        add_author_metadata(zenodo_metadata,
+                            codemeta_metadata[person],
+                            person
+                            )
+
+    assert 'creators' in zenodo_metadata.keys()
+    # 4 'creators' one repeated, should not be duplicated.
+    # Maintainer and Contributor. Author and Creator are the same person
+    assert len(zenodo_metadata['creators']) == 3
+
+    assert 'contributors' in zenodo_metadata.keys()
+    # Editor, Producer, Publisher, Provider and Sponsor
+    assert len(zenodo_metadata['contributors']) == 5
+
+
+def test_parse_person_schema_property():
+    from eossr.metadata.codemeta2zenodo import \
+        parse_person_schema_property, \
+        codemeta_contributors_fields
+
+    with open(join(SAMPLES_DIR, "codemeta_contributors_sample.json")) as f:
+        codemeta_metadata = json.load(f)
+
+    for person in codemeta_contributors_fields:
+        zenodo_metadata = parse_person_schema_property(codemeta_metadata[person],
+                                                       person)
+        if person == 'editor':
+            assert zenodo_metadata['type'] == 'Editor'
+        elif person == 'producer':
+            assert zenodo_metadata['type'] == 'Producer'
+        elif person == 'sponsor':
+            assert zenodo_metadata['type'] == 'Sponsor'
+        else:
+            assert zenodo_metadata['type'] == 'Other'
+
+
+class TestConverting(unittest.TestCase):
+    def test_sample_file_conversion(self):
+        from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
+
+        outfile = tempfile.NamedTemporaryFile(delete=True)
+        parse_codemeta_and_write_zenodo_metadata_file(
+            join(SAMPLES_DIR, "codemeta_sample1.json"), outfile.name
+        )
+
+        json.load(outfile)
+
+    def test_root_codemeta_conversion(self):
+        from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
+
+        outfile = tempfile.NamedTemporaryFile(delete=True)
+        parse_codemeta_and_write_zenodo_metadata_file(
+            join(ROOT_DIR, "codemeta.json"), outfile.name
+        )
+        json.load(outfile)
+
+
+class TestLicense(unittest.TestCase):
+    def test_license1(self):
+        from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController
+
+        converter = CodeMeta2ZenodoController()
+        converter.convert_license()
+        assert 'license' not in converter.zenodo_data
+
+        converter.codemeta_data['license'] = 'https://spdx.org/licenses/MIT'
+        converter.convert_license()
+        assert converter.zenodo_data['license'] == 'MIT'
+
+        converter.codemeta_data['license'] = './COPYING'
+        converter.convert_license()
+        assert converter.zenodo_data['license'] == 'other-open'

eossr/scripts/eossr_codemeta2zenodo.py

@@ -4,7 +4,7 @@ import sys
 import argparse
 from pathlib import Path
 from distutils.util import strtobool
-from codemeta2zenodo.crosswalk.codemeta2zenodo_crosswalk import parse_codemeta_and_write_zenodo_metadata_file
+from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
 
 
 def query_yes_no(question, default="yes"):

eossr/scripts/tests/test_eossr_codemeta2zenodo.py

+import subprocess
+from os.path import dirname, realpath, join
+
+ROOT_DIR = dirname(realpath("codemeta.json"))
+
+
+def run_script(*args):
+    result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+
+    if result.returncode != 0:
+        raise ValueError(
+            f"Running {args[0]} failed with return code {result.returncode}"
+            f", output: \n {result.stdout}"
+        )
+
+
+def test_codemeta2zenodo():
+    run_script("eossr-codemeta2zenodo", "-i", join(ROOT_DIR, "codemeta.json"))