Commit 5e2c5ee8 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

incorporate the last modifications - test the connection to zenodo - developat the zenodoCI lib

parent 035c25bd
Pipeline #93406 passed with stage
in 2 minutes and 13 seconds
......@@ -72,20 +72,20 @@ deploy_zenodo:
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $SANDBOX_ZENODO_TOKEN
# --sandbox_zenodo True
# --sandbox True
# --input-directory ./build
### 4 - To deploy a NEW DEPOSIT to ZENODO
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $ZENODO_TOKEN
# --sandbox_zenodo False
# --sandbox False
# --input-directory ./build
### 4 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
--sandbox_zenodo False
--sandbox False
--token $ZENODO_TOKEN
--deposit_id $DEPOSIT_ID_ESCAPE_TEMPLATE
......
# README _.zenodoci_ library
# README _.zenodoci_
**PLEASE HAVE A LOOK TO THE LICENSING SECTION BELOW BEFORE IMPLEMENTING ANY PART OF THIS CODE INTO YOURS !!**
## Continuous Deployment to Zenodo
Library to manage an upload to Zenodo through its REST API.
## _Quickstart_
1. Add a `codementa.json` file to your project.
2. Create a token at (sandbox)zenodo and add it as a variable to your project (see below).
3. **TEST** that you can communicate correctly with Zenodo. You will test and debug at the same time that all the
stages of the GitLabCI-Zenodo pipeline will work correctly.
```bash
$ python .zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
```
## Continuous Deployment to Zenodo
The source code contained in this folder is based on the [ZenodoCI](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci)
project. The library was developed specifically to perform a deploy stage (to the Zenodo repository) in a GitLab CI
pipeline that **could be implemented in any external project**.
The library (composed of the scripts within the `.zenodoci` directory) provides a module to handle the upload of
specified file(s) to (sandbox)zenodo. Please **JUST** upload stable versions/releases of source code and/or image
specified file(s) to the [ESCAPE2020 Zenodo community](https://zenodo.org/communities/escape2020/). Please **JUST** upload stable versions/releases of source code and/or image
containers!
A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the
GitLabCI pipeline.
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library and
the built Singularity container created in the previous CI stage (check the `.gitlabci` directory too) to:
......@@ -58,7 +75,7 @@ your personal token, you should create an environment variable in your GitLab re
The environment variable will look like this:
```sh
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox_zenodo False
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
```
## License of the `template_project_repository`:
......
#!/usr/bin/env python
# E. Garcia Nov 2020
# Module to test the connection and the upload of new entries/version to Zenodo.
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
if __name__ == '__main__':
# Required arguments
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
args = parser.parse_args()
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag
)
zenodo.test_upload_to_zenodo()
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
......@@ -26,8 +30,8 @@ def create_zenodo_metadata(metadata_filename):
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
......@@ -43,16 +47,16 @@ def create_zenodo_metadata(metadata_filename):
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
......@@ -63,13 +67,13 @@ if __name__ == '__main__':
args = parser.parse_args()
z = ZenodoAPI(
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = z.create_new_entry()
new_entry = zenodo.create_new_entry()
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
......@@ -82,7 +86,7 @@ if __name__ == '__main__':
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(
new_upload = zenodo.upload_file_entry(
deposition_id,
name_file=file,
path_file=full_path_file
......@@ -99,7 +103,7 @@ if __name__ == '__main__':
entry_metadata = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(
update_entry = zenodo.update_metadata_entry(
deposition_id,
data=entry_metadata
)
......@@ -111,9 +115,9 @@ if __name__ == '__main__':
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = z.publish_entry(deposition_id)
# publish = zenodo.publish_entry(deposition_id)
# print(publish.json())
print("New deposit correctly published !")
print(f"The new doi should look like 10.5281/{deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
......@@ -26,8 +30,8 @@ def create_zenodo_metadata(metadata_filename):
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
......@@ -43,17 +47,16 @@ def create_zenodo_metadata(metadata_filename):
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
......@@ -69,13 +72,13 @@ if __name__ == '__main__':
args = parser.parse_args()
z = ZenodoAPI(
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - request a new version of an existing deposit
new_version = z.new_version_entry(args.deposit_id)
new_version = zenodo.new_version_entry(args.deposit_id)
if new_version.status_code < 399:
print(f"Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !")
......@@ -87,7 +90,7 @@ if __name__ == '__main__':
# PRE-2 - If you DO NOT want to erase the old files, comment the following lines
old_files_ids = [file['id'] for file in new_version.json()['files']]
for file_id in old_files_ids:
z.erase_file_entry(
zenodo.erase_file_entry(
new_deposition_id,
file_id
)
......@@ -96,7 +99,7 @@ if __name__ == '__main__':
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(
new_upload = zenodo.upload_file_entry(
new_deposition_id,
name_file=file,
path_file=full_path_file
......@@ -112,7 +115,7 @@ if __name__ == '__main__':
update_entry_metadata = json.load(json_file)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(
update_entry = zenodo.update_metadata_entry(
new_deposition_id,
data=update_entry_metadata
)
......@@ -130,4 +133,4 @@ if __name__ == '__main__':
print("New version of the old deposition correctly published !")
print(f"Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}")
print(f"The new doi should look like 10.5281/{new_deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
#!/usr/bin/env python
# E. Garcia Nov 20
# email: garcia 'at' lapp.in2p3.fr
import os
import sys
import json
......@@ -31,7 +36,8 @@ def parse_person_schema_property(person_property, contributor_field):
zenodo_person['name'] = name
if "@id" in person_property:
if 'orcid.org/' in person_property["@id"]: # "https://orcid.org/0000-0002-5686-2078" format not accepted
if 'orcid.org/' in person_property["@id"]:
# "https://orcid.org/0000-0002-5686-2078" codemeta format not accepted
zenodo_person['orcid'] = person_property["@id"].split('orcid.org/')[-1]
else:
zenodo_person['orcid'] = person_property["@id"]
......@@ -155,7 +161,7 @@ def find_matching_metadata(codemeta_json):
metadata_zenodo['keywords'] = [codemeta_json['keywords']]
if 'license' in codemeta_json:
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1] # TODO to be improved
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1]
if 'releaseNotes' in codemeta_json:
metadata_zenodo['notes'] = "Release Notes: " + codemeta_json['releaseNotes']
if 'citation' in codemeta_json:
......
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# E. Garcia 2020
# email: garcia 'at' lapp.in2p3.fr
import json
import requests
import numpy as np
from utils_zenodoci import (find_root_directory,
parse_codemeta_and_write_zenodo_metadata_file
)
class ZenodoAPI:
......@@ -9,14 +16,17 @@ class ZenodoAPI:
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
six tasks within the (sandbox.)zenodo api environment:
the following tasks within the (sandbox.)zenodo api environment:
- Create a new deposit,
- Create a new version of an existing deposit,
- Upload files to a specific Zenodo entry,
- Erase (old version) files from an entry (when creating a new_version entry and uploading new_version files),
- Upload information to the entry (Zenodo compulsory deposit information),
- Publish an entry
- Fetches a user's published entries,
- Creates a new deposit,
- Creates a new version of an existing deposit,
- Uploads files to a specific Zenodo entry,
- Erases a non-published entry / new version draft,
- Erases (old version) files from an entry (when creating a new_version entry and uploading
new_version files),
- Uploads information to the entry (Zenodo compulsory deposit information),
- Publishes an entry
:param access_token: str
......@@ -32,13 +42,33 @@ class ZenodoAPI:
self.zenodo_api_url = zenodo_api_url
self.access_token = access_token
self.exist_codemeta_file = False
self.path_codemeta_file = ''
self.exist_zenodo_metadata_file = False
self.path_zenodo_metadata_file = ''
def fetch_user_entries(self):
"""
Fetch the published entries of an user. Works to tests connection to Zenodo too.
GET method to {zenodo_api_url}/deposit/depositions
:return: obj
request.get answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
parameters = {'access_token': self.access_token}
return requests.get(url, params=parameters)
def create_new_entry(self):
"""
Create a new entry / deposition in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
headers = {"Content-Type": "application/json"}
......@@ -50,6 +80,7 @@ class ZenodoAPI:
"""
Upload a file to a Zenodo entry. If first retrieve the entry by a GET method to the
{zenodo_api_url}/deposit/depositions/{entry_id}.
PUT method to {bucket_url}/{filename}. The full api url is recovered when the entry is firstly retrieved.
:param entry_id: str
......@@ -59,7 +90,8 @@ class ZenodoAPI:
:param path_file: str
Path to the file to be uploaded
:return: json requests.put object
:return: obj
json requests.put object
"""
# 1 - Retrieve and recover information of an existing deposit
parameters = {'access_token': self.access_token}
......@@ -75,10 +107,11 @@ class ZenodoAPI:
return upload.json()
def update_info_entry(self, entry_id, data):
def update_metadata_entry(self, entry_id, data):
"""
Update an entry resource. Data should be the entry information that will be shown when a deposition is visited
at the Zenodo site.
PUT method to {zenodo_api_url}/deposit/depositions/{entry_id}. `data` MUST be included as json.dump(data)
:param entry_id: str
......@@ -86,13 +119,33 @@ class ZenodoAPI:
:param data: object
json object containing the metadata (compulsory fields) that are enclosed when a new entry is created.
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
headers = {"Content-Type": "application/json"}
parameters = {'access_token': self.access_token}
return requests.put(url, data=json.dumps(data), headers=headers, params=parameters)
return requests.put(url, data=json.dumps(data),
headers=headers, params=parameters)
def erase_entry(self, entry_id):
"""
Erase an entry/new version of an entry that HAS NOT BEEN published yet.
Any new upload/version will be first saved as 'draft' and not published until confirmation (i.e, requests.post)
DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}.
:param entry_id: str
deposition_id of the Zenodo entry to be erased
:return: obj
requests.delete json answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
parameters = {'access_token': self.access_token}
return requests.delete(url, params=parameters)
def erase_file_entry(self, entry_id, file_id):
"""
......@@ -106,7 +159,8 @@ class ZenodoAPI:
:param file_id: str
Id of the files stored in Zenodo
:return: requests.delete answer
:return: obj
requests.delete answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}"
parameters = {'access_token': self.access_token}
......@@ -121,7 +175,8 @@ class ZenodoAPI:
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish"
parameters = {'access_token': self.access_token}
......@@ -136,9 +191,120 @@ class ZenodoAPI:
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.post answer
:return: obj
requests.post answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion"
parameters = {'access_token': self.access_token}
return requests.post(url, params=parameters)
def search_codemeta_file(self):
"""Check if a `codemeta.json` files exists in the ROOT directory of the project"""
root_dir = find_root_directory()
codemeta_file = root_dir / 'codemeta.json'
if codemeta_file.exists():
print("\n * Found codemeta.json file within the project !")
self.exist_codemeta_file = True
self.path_codemeta_file = codemeta_file
else:
print("\n ! codemeta.json file NOT found in the root directory of the project !")
def search_zenodo_json_file(self):
"""Check if a `.zenodo.json` files exists in the ROOT directory of the project"""
root_dir = find_root_directory()
zenodo_metadata_file = root_dir / '.zenodo.json'
if zenodo_metadata_file.exists():
print("\n * Found .zenodo.json file within the project !")
self.exist_zenodo_metadata_file = True
self.path_codemeta_file = zenodo_metadata_file
else:
print("\n ! .zenodo.json file NOT found in the root directory of the project !")
def test_upload_to_zenodo(self):
"""
`Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every
stage is the correct one.
Tests:
- If it exists a `codemeta.json` file
- If it exists a `.zenodo.json` file
- If not, it creates one, based on the `codemeta.json` file
- The communication with Zenodo through its API to verify that:
- You can fetch an user entries
- You can create a new entry
- The provided zenodo metadata can be digested, and not errors appear
- Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED !
"""
# Search for codemeta.json file within the project
self.search_codemeta_file()
if self.exist_codemeta_file:
# Search for zenodo.json metadata file within the project
self.search_zenodo_json_file()
if self.exist_zenodo_metadata_file:
print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
else:
print("\n ! Creating a .zenodo.json file from your codemeta.json file...\n"
" Please add, commit and push this file to your project repository.")
self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
self.path_zenodo_metadata_file)
# 1 - Test connection
print("Testing communication with Zenodo...")
test_connection = self.fetch_user_entries()
try:
np.testing.assert_equal(test_connection.status_code, 200)
print("OK !")
except:
print(test_connection.json())
# 2 - Test new entry
print("Testing the creation of a dummy entry to (sandbox)Zenodo...")
new_entry = self.create_new_entry()
try:
np.testing.assert_equal(new_entry.status_code, 201)
print("OK !")
except:
print(new_entry.json())
# 3 - Test upload metadata
print("Testing the ingestion of the Zenodo metadata...")
test_entry_id = new_entry.json()['id']
with open(self.path_zenodo_metadata_file) as file:
metadata_entry = json.load(file)
update_metadata = self.update_metadata_entry(test_entry_id,
data=metadata_entry
)
try:
np.testing.assert_equal(update_metadata.status_code, 200)
print("OK !")
except:
print(update_metadata.json())
# 4 - Test delete entry
print("Testing the deletion of the dummy entry...")
delete_test_entry = self.erase_entry(test_entry_id)
try:
np.testing.assert_equal(delete_test_entry.status_code, 204)
print("OK !")
except:
print(delete_test_entry.json())
print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
"You should not face any trouble when uploading a project to Zenodo through the "
"ESCAPE-GitLabCI pipeline.\n"
"In case you do, please contact us !\n")
else:
print("\n ! Please add a codemeta.json file to the ROOT directory of your project.\n")
# -*- coding: utf-8 -*-
#
#!/usr/bin/env python
# Enrique Garcia. Nov 2020.
# email: garcia 'at' lapp.in2p3.fr
......@@ -160,7 +160,7 @@ def find_matching_metadata(codemeta_json):
metadata_zenodo['keywords'] = [codemeta_json['keywords']]
if 'license' in codemeta_json:
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1] # TODO to be improved
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1]
if 'releaseNotes' in codemeta_json:
metadata_zenodo['notes'] = "Release Notes: " + codemeta_json['releaseNotes']
if 'citation' in codemeta_json:
......
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import sys
import yaml
import os.path
......@@ -89,7 +89,7 @@ class FillCiScript:
f" - export REPOSITORY_BASE_URL={self.repository_info['base_url']}$REPOSITORY_NAME\n",
"\n",
" - mkdir -p build\n",
" - /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_URL\n",
" - /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL\n",
"\n",
" - ls ./build\n",
"\n"
......@@ -108,7 +108,7 @@ class FillCiScript:
" - >\n",
" python3 .zenodoci/upload_new_deposit.py\n",
" --token $ZENODO_TOKEN\n",
" --sandbox_zenodo False\n",
" --sandbox False\n",