Commit 72692061 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Merge branch 'create_package' into 'master'

Create package

See merge request !9
parents 9c30abb0 563318dc
Pipeline #98405 passed with stages
in 2 minutes and 31 seconds
stages:
- test
- deploy
test_install:
stage: test
image: python:3.6.11-buster
script:
- apt-get -y update
- pip install .
only:
- branches
deploy_zenodo:
stage: deploy
image: python:3.6.11-buster
......@@ -16,11 +26,12 @@ deploy_zenodo:
#- cat /etc/os-release # Debian GNU/Linux 10 (buster)
#- pip3 --version # pip 20.1.1
#- python3 --version # 3.6.11 as expected
- pip3 install requests numpy
- pip install requests numpy
- pip install . # Install ZenodoCI
### 2 - Test the communication to Zenodo - Uncomment if needed, or if already done it manually.
- python3 .zenodoci/test_connection_zenodo.py -t $SANDBOX_ZENODO_TOKEN -s True
#- python3 .zenodoci/test_connection_zenodo.py -t $ZENODO_TOKEN -s False
- test_connection_zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -r $CI_PROJECT_DIR
#- test_connection_zenodo -t $ZENODO_TOKEN -s False --repo_dir $CI_PROJECT_DIR
script:
### 3 - Get the last tag/release of the repository
......@@ -30,26 +41,23 @@ deploy_zenodo:
### 4 - Download the repository and move it to the build directory
### If no release is found/correctly parsed, the script will download the last commit pushed to the master branch
- mkdir -p build
- /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL
- /bin/bash zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL
- ls ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
- >
python3 .zenodoci/upload_new_deposit.py
--token $SANDBOX_ZENODO_TOKEN
--sandbox True
--input-directory ./build
upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-directory ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO
#- >
# python3 .zenodoci/upload_new_deposit.py
# upload_new_deposit
# --token $ZENODO_TOKEN
# --sandbox False
# --input-directory ./build
### 5 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
upload_new_version_deposit
--token $SANDBOX_ZENODO_TOKEN
--sandbox True
--input-directory ./build
......
#!/usr/bin/env python
# E. Garcia Nov 20
# email: garcia 'at' lapp.in2p3.fr
import os
import sys
import json
from pathlib import Path
from distutils.util import strtobool
def parse_person_schema_property(person_property, contributor_field):
"""
Parse the Person Schema property correctly
Parameters:
--------
person_property: dict
dictionary codemeta key with the a list or a single Person property item.
contributor_field : str
contributor type {'editor', 'producer', 'sponsor'} or publisher, although the last one can only happen if
`upload_type` is publication (NOT SUPPORTED - contact E. Garcia by email).
Returns:
--------
zenodo_person: dict
dictionary with the correct zenodo syntax for all {author, contributor, maintainer}.
"""
zenodo_person = {}
special_contributor_cases = ['editor', 'producer', 'publisher', 'provider', 'sponsor']
name = person_property['familyName']
if 'givenName' in person_property:
name += f', {person_property["givenName"]}'
zenodo_person['name'] = name
if "@id" in person_property:
if 'orcid.org/' in person_property["@id"]:
# "https://orcid.org/0000-0002-5686-2078" codemeta format not accepted
zenodo_person['orcid'] = person_property["@id"].split('orcid.org/')[-1]
else:
zenodo_person['orcid'] = person_property["@id"]
if "affiliation" in person_property:
zenodo_person['affiliation'] = person_property['affiliation']['name']
# Parse correctly the contributors
if contributor_field in special_contributor_cases:
if contributor_field is 'provider' or contributor_field is 'publisher':
zenodo_person['type'] = 'Other'
else:
try:
zenodo_person['type'] = person_property["type"]
except:
zenodo_person['type'] = contributor_field
return zenodo_person
def add_author_metadata(zenodo_file, codemt_file, field):
"""
Aux function to parse correctly all the authors, contributors and maintainers that can be found at the
codemeta.json file
zenodo_file: dict
metadata dictionary with the zenodo syntax
codem_file: list or dict
metadata dictionary key field with the codemeta syntax
field: str
codemeta key field specifying creator {author, contributor, maintainer, creator}, or
contributors {editor, sponsor, producer, project manager...}
"""
full_contacts = {}
creators_fields = ['author', 'creator', 'maintainer', 'contributor']
contributors_fields = ['editor', 'producer', 'publisher', 'provider', 'sponsor']
# First create the full contact agenda by field
if type(codemt_file[field]) is list:
for person_property in codemt_file[field]:
zenodo_person = parse_person_schema_property(person_property, field)
# 'name' is the only key that MUST be contained in a person_property at least
full_contacts[zenodo_person['name']] = zenodo_person
else:
zenodo_person = parse_person_schema_property(codemt_file[field], field)
full_contacts[zenodo_person['name']] = zenodo_person
# then save each person by field and avoid duplicates
for i, person in enumerate(full_contacts):
if field in creators_fields:
# Contributors and maintainers in the same zenodo key
if i == 0 and 'creators' not in zenodo_file:
zenodo_file['creators'] = []
elif person not in zenodo_file['creators']:
zenodo_file['creators'].append(full_contacts[person])
else:
pass # avoid duplicates
elif field in contributors_fields:
if i == 0 and 'contributors' not in zenodo_file:
zenodo_file['contributors'] = []
elif person not in zenodo_file['contributors']:
zenodo_file['contributors'].append(full_contacts[person])
else:
pass # avoid duplicates
def find_matching_metadata(codemeta_json):
"""
Please note that the following fields are ASSUMED. If they are not correct, change them, or contact us otherwise.
"access_right": "open"
"language": "eng"
param codemeta_json: dict
already parsed dictionary containing the metadata of the codemeta.json file
Returns:
--------
metadata_zenodo : dict
dictionary cotaining the metadata information found at the codemeta.json file but written using the Zenodo
syntax.
"""
person_filed = ['author', 'creator', 'maintainer', 'contributor', 'editor', 'producer', 'publisher',
'provider', 'sponsor']
metadata_zenodo = {'language': 'eng',
'access_right': 'open'}
if codemeta_json["@type"] == "SoftwareSourceCode":
metadata_zenodo['upload_type'] = 'software'
else:
metadata_zenodo['upload_type'] = ''
print("\nCould not identify the type of schema in the `codemeta.json file`.\n"
"Thus the 'upload_type' within the `.zenodo.json` file was left EMPTY.\n"
"Please fill it up by yourself - otherwise zenodo will NOT be able to publish your entry.\n")
if 'name' in codemeta_json:
metadata_zenodo['title'] = codemeta_json['name']
if 'description' in codemeta_json:
metadata_zenodo['description'] = codemeta_json['description']
if 'softwareVersion' in codemeta_json and 'version' not in codemeta_json:
metadata_zenodo['version'] = codemeta_json['softwareVersion']
elif 'version' in codemeta_json and 'softwareVersion' not in codemeta_json:
metadata_zenodo['version'] = codemeta_json['version']
else:
metadata_zenodo['version'] = codemeta_json['version']
if 'keywords' in codemeta_json:
if type(codemeta_json['keywords']) == list:
metadata_zenodo['keywords'] = codemeta_json['keywords']
else:
metadata_zenodo['keywords'] = [codemeta_json['keywords']]
if 'license' in codemeta_json:
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1]
if 'releaseNotes' in codemeta_json:
metadata_zenodo['notes'] = "Release Notes: " + codemeta_json['releaseNotes']
if 'citation' in codemeta_json:
metadata_zenodo['references'] = codemeta_json['citation']
if 'datePublished' in codemeta_json:
metadata_zenodo['publication_date'] = codemeta_json['datePublished']
for person in person_filed:
if person in codemeta_json:
add_author_metadata(metadata_zenodo, codemeta_json, field=person)
return metadata_zenodo
def add_compulsory_escape_metadata(json_file):
"""
Add compulsory information to the .zenodo.json file:
* zenodo community : ESCAPE2020
* ESCAPE grant ID (zenodo syntax)
param json_file: dict
dictionary containing the .zenodo.json metadata information
"""
json_file["communities"] = [{"identifier": "escape2020"}]
json_file["grants"] = [{"id": "10.13039/501100000780::824064"}]
def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, zenodo_outname):
"""
Reads the codemeta.json file and creates a new `.zenodo.json` file. This file will contain the SAME information
that in the codemeta.json file but *** WITH THE ZENODO SYNTAX. ***
codemeta_filename: str or Path
path to the codemeta.json file
zenodo_outname: str or Path
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
with open(codemeta_filename) as infile:
codemeta_json = json.load(infile)
metadata_zenodo = find_matching_metadata(codemeta_json)
add_compulsory_escape_metadata(metadata_zenodo)
# Correct format for Zenodo
data = {'metadata': metadata_zenodo}
with open(zenodo_outname, 'w') as outfile:
json.dump(data, outfile, indent=4, sort_keys=True)
def query_yes_no(question, default="yes"):
"""
Ask a yes/no question via raw_input() and return their answer.
:param question: str
question to the user
:param default: str - "yes", "no" or None
resumed answer if the user just hits <Enter>.
"yes" or "no" will set a default answer for the user
None will require a clear answer from the user
:return: bool - True for "yes", False for "no"
"""
valid = {"yes": True, "y": True, "ye": True,
"no": False, "n": False}
if default is None:
prompt = " [y/n] "
elif default == "yes":
prompt = " [Y/n] "
elif default == "no":
prompt = " [y/N] "
else:
raise ValueError("invalid default answer: '%s'" % default)
while True:
sys.stdout.write(question + prompt)
choice = input().lower()
if default is not None and choice == '':
return valid[default]
else:
try:
return bool(strtobool(choice))
except:
sys.stdout.write("Please respond with 'yes' or 'no' "
"(or 'y' or 'n').\n")
def query_continue(question, default="no"):
"""
Ask a question and if the answer is no, exit the program.
Calls `query_yes_no`.
:param question: str
:param default: str
:return answer: bool - answer from query_yes_no
"""
answer = query_yes_no(question, default=default)
if not answer:
sys.exit("Program stopped by user")
else:
return answer
def find_root_directory():
"""
Find root directory of the project. This library MUST be added to the root directory of the same, i.e., MUST be
a subdirectory of the root dir.
:return root_directory: obj
Path object with root directory
"""
current_dir = os.path.abspath(os.path.dirname(__file__))
current_dir_name = current_dir.split('/')[-1]
root_directory = Path(str(current_dir.split(current_dir_name)[0]))
return root_directory
......@@ -7,6 +7,15 @@ https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/commits/master)
Library to manage an upload to Zenodo through its REST API.
## Install
```sh
$ git clone https://gitlab.in2p3.fr/escape2020/wp3/zenodoci.git
$ cd zenodoci
$ pip install .
```
## _Quickstart_ - Test the full OSSR-CI and the communication to Zenodo
1. Add a `codementa.json` file to your project.
......@@ -15,7 +24,7 @@ Library to manage an upload to Zenodo through its REST API.
3. **TEST** that you can communicate correctly with Zenodo. You will test and debug at the same time that all the
stages of the GitLabCI-Zenodo pipeline will work correctly.
```bash
$ python .zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
$ python zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
```
## Continuous Integration to Zenodo
......@@ -57,7 +66,7 @@ Adding a single `codemeta.json` file to the root directory of your project will
[ESCAPE metadata template](https://gitlab.in2p3.fr/escape2020/wp3/escape_metadata_template) project for a _quickstart_ on
how to easily create a `codemeta.json` file.
Last but not least ! Please note that during the CI pipeline the `.zenodoci` module will search for a `codemeta.json` file
Last but not least ! Please note that during the CI pipeline the `zenodoci` module will search for a `codemeta.json` file
and will automatically create the equivalent file to provide metadata to the Zenodo repository. The `.zenodo.json` file
will contain the exactly same information that in the `codemeta.json` file but using the Zenodo syntax.
......@@ -79,7 +88,7 @@ your personal token, you should create an environment variable in your GitLab re
The environment variable will look like this:
```sh
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
$ python zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
```
# Contact
......
#!/usr/bin/env python
import re
from setuptools import setup, find_packages
entry_points = {'console_scripts': [
'upload_new_deposit = zenodoci.upload_new_deposit:main',
'upload_new_version_deposit = zenodoci.upload_new_version_deposit:main',
'test_connection_zenodo = zenodoci.test_connection_zenodo:main'
]
}
def get_property(prop, project):
result = re.search(r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop), open(project + '/__init__.py').read())
return result.group(1)
setup(
name='zenodoci',
version=get_property('__version__', 'zenodoci'),
description="REST API handler to manage the uploads and communication with Zenodo.",
install_requires=[
'requests',
'codemeta2zenodo_crosswalk @ git+https://gitlab.in2p3.fr/escape2020/wp3/codemeta2zenodo',
],
packages=find_packages(),
# tests_require=['pytest'],
author='Enrique Garcia',
author_email='garcia<at>lapp.in2p3.fr',
url='https://gitlab.in2p3.fr/escape2020/wp3/zenodoci',
license='MIT',
entry_points=entry_points
)
# ZenodoCI
**PLEASE HAVE A LOOK TO THE LICENSING SECTION BELOW BEFORE IMPLEMENTING ANY PART OF THIS CODE INTO YOURS !!**
## Continuous Deployment to Zenodo
The source code contained in this folder is based on the
[zenodo-python repository](https://github.com/SiLeBAT/zenodo-python).
The software (composed of the `zenodolib.py` script) provides a library to handle the upload of a
specified file(s) to (sandbox)zenodo. Please **JUST** upload stable versions/releases of source code and/or images
containers!
By using the `zenodolib` library, as well as the created Singularity container (check the `.gitlabci` directory),
the second stage of the CI pipeline (see the `.gitlab-ci.yml` file) will:
- Either upload the desired file(s) to the ESCAPE community in Zenodo.
- Either upload a new version of an existing entry to Zenodo.
The `repository_information.json` file must be filled up before pushing to the project repository. Also, depending on
the case, the corresponding python script (`upload_new_deposit.py` or `upload_new_version_of_deposit.py`) must be
adapted and included into the `.gitlab-ci.yml` file with its corresponding arguments.
### Zenodo token & GitLab CI environment variable
To connect the GitLab repository with Zenodo in an autonomous way, a personal access token must be created. This token
is assign to a **single** Zenodo account, and it will allow to interact with (sandbox)zenodo through its
API. To create the token:
- Go to (sandbox)zenodo.org
- Account --> Applications --> Personal access token --> New token.
This token will be passed later in the continuous deployment stage of the CI/CD pipeline. For not sharing publicly
your personal token, you should create an environment variable in your GitLab repository. This way, the token could be used as a variable without
revealing its value. To create an an environment variable:
- Go to your GitLab repository.
- Settings --> CI/CD --> Variables --> Add variable --> Fill the fields --> Mask your variable(s) !!
The environment variable will look like this:
```sh
$ python zenodoci/upload_new_deposit.py -i build/Singularity -t $ZENODO_TOKEN
```
## License of the repository
The license of the 'parent' repository is `GNU GENERAL PUBLIC LICENSE Version 3` ("GNU GPL v3" or "GPLv3"). Before
implementing anything from the current or the original repository **please check that the license of the project
in which you are going to implement these source files is compatible with the GPLv3 license**.
If this is the case:
- The whole resulting project must be distributed with the same license, i.e., `GPLv3` and contain the license
and copyright notice (see the LICENSE file of the `template_project_repository`). GPLv3 is copyleft.
- Changes must be stated (here is an example of the kind of text that must be included in each modified file):
> This code is part of https://github.com/SiLeBAT/zenodo-python
>
> Copyright (c) 2015 Federal Institute for Risk Assessment (BfR), Germany
> This program is free software: you can redistribute it and/or modify
> it under the terms of the GNU General Public License as published by
> the Free Software Foundation, either version 3 of the License, or
> (at your option) any later version.
>
> This program is distributed in the hope that it will be useful,
> but WITHOUT ANY WARRANTY; without even the implied warranty of
> MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> GNU General Public License for more details.
>
> You should have received a copy of the GNU General Public License
> along with this program. If not, see <http://www.gnu.org/licenses/>.
### Specific case of the `template_project_repository`:
The `template_project_repository` contains code from different projects. This 'mixing' can be done because:
- Both 'parent' projects are Open Source.
- Both licenses, `BSD 3-Clause` and `GPLv3`, are [compatible](
https://www.gnu.org/licenses/gpl-faq.html#WhatDoesCompatMean); this means that source code distributed with a BSD
3-Clause license can be included within a project with a GPLv3 license:
- Note however that the GNU General Public License v3 is 'copyleft', meaning that the resulting project from the use
of any code licensed with GPLv3, must be distributed with the same license, thus GPLv3.
This are the reasons why the `template_project_repository` is distributed with the GNU General Public License Version
3 and some files contain a BSD-3 Clause license.
**PLEASE TAKE THE TIME TO CHECK AND VERIFY LICENSES AND THEIR COMPATIBILITIES**
\ No newline at end of file
__version__ = "1.1"
......@@ -5,10 +5,10 @@
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from zenodoci.zenodoapi import ZenodoAPI
if __name__ == '__main__':
def main():
# Required arguments
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
......@@ -24,10 +24,22 @@ if __name__ == '__main__':
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--repo_dir', '-r', action='store',
dest='repository_dir',
help='Path to the root directory of the directory to be uploaded.'
'DEFAULT; assumed to be on it, i.e., "./"',
default='./'
)
args = parser.parse_args()
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag
sandbox=args.sandbox_flag,
repo_root_dir=args.repository_dir
)
zenodo.test_upload_to_zenodo()
if __name__ == '__main__':
main()
......@@ -6,25 +6,26 @@
import os
import json
import argparse
from pathlib import Path
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from utils_zenodoci import (parse_codemeta_and_write_zenodo_metadata_file,
find_root_directory
)
from zenodoci.zenodoapi import ZenodoAPI
from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
def create_zenodo_metadata(metadata_filename):
def create_zenodo_metadata(metadata_filename, repo_root_dir='./'):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
param metadata_filename: str
path and name to the zenodo metada json file
path and name to the zenodo metadata json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
root_dir = find_root_directory()
# root_dir = find_root_directory()
root_dir = Path(repo_root_dir)
print(f'working dir : {root_dir}') # DEBUG
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(files_json)
print(f'JSON files found {files_json}')
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
......@@ -44,7 +45,7 @@ def create_zenodo_metadata(metadata_filename):
exit(-1)
if __name__ == '__main__':
def main():
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
parser.add_argument('--token', '-t', type=str,
......@@ -78,8 +79,9 @@ if __name__ == '__main__':
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
doi = new_entry.json()['metadata']['prereserve_doi']['doi']
print(f"Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
print(f" * Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
else:
print(f" ! ERROR; the NEW entry COULD NOT be created.")
print(new_entry.json())
# 2 - upload files
......@@ -92,7 +94,7 @@ if __name__ == '__main__':
path_file=full_path_file
)
print(f"File {file} correctly uploaded !\n", new_upload)
print(f" * File {file} correctly uploaded !\n", new_upload)
# 3 - Create the zenodo metadata file from a codemeta.json file
zenodo_metadata_filename = '.zenodo.json'
......@@ -109,15 +111,19 @@ if __name__ == '__main__':
)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !")
print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !")
else:
print(f"Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
print(f" ! Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",