Commit 823ad7b1 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Merge branch 'connection_test_zenodoapi' into 'master'

integrate test_connection_zenodo to the ZenodoAPI

Closes #3

See merge request !5
parents 4dc86dca 1ef7189c
Pipeline #93404 passed with stage
in 36 seconds
......@@ -18,12 +18,12 @@ deploy_zenodo:
#- python3 --version # 3.6.11 as expected
- pip3 install requests numpy
### 2 - Check that you can correctly communicate with (sandobox)zenodo - Uncomment if needed.
### 2 - Test the communication to Zenodo - Uncomment if needed, or if already done it manually.
- python3 .zenodoci/test_connection_zenodo.py -t $SANDBOX_ZENODO_TOKEN -s True
#- python3 .zenodoci/test_connection_zenodo.py -t $ZENODO_TOKEN -s False
script:
### 3 - Get the last tag/release of the repository
- export REPOSITORY_NAME=zenodoci
- export REPOSITORY_BASE_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
......@@ -33,30 +33,30 @@ deploy_zenodo:
- /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_URL
- ls ./build
script:
### 5 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
- >
python3 .zenodoci/upload_new_deposit.py
--token $SANDBOX_ZENODO_TOKEN
--sandbox_zenodo True
--sandbox True
--input-directory ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $ZENODO_TOKEN
# --sandbox_zenodo False
# --sandbox False
# --input-directory ./build
### 5 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
--token $SANDBOX_ZENODO_TOKEN
--sandbox_zenodo True
--sandbox True
--input-directory ./build
--deposit_id $DEPOSIT_ID_ZENODOCI
# --token $ZENODO_TOKEN
# --sandbox_zenodo False
# --sandbox False
only:
### Ideally this stage should be run only when a new release / tag of the source code is created, i.e., (- tags).
# The script is changed to check that the both `upload_new_deposit` and `upload_new_version_deposit` works nicely.
......
from .zenodoapi import *
from .utils_zenodoci import *
from .upload_new_deposit import *
from .upload_new_version_deposit import *
#!/usr/bin/env python
# E. Garcia Nov 2020
# Module to test the connection and the upload of new entries/version to Zenodo.
import argparse
import requests
import numpy as np
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
......@@ -13,38 +14,20 @@ if __name__ == '__main__':
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
args = parser.parse_args()
z = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# Test that the you can communicate with (sandbox)zenodo - you have passed the correct token
parameters = {'access_token': z.access_token}
test_connection = requests.get(
f'{z.zenodo_api_url}/deposit/depositions',
params=parameters
)
np.testing.assert_equal(test_connection.status_code, 200)
# Test that you can create a new entry - then all the rest of the pipeline will work
new_entry = z.create_new_entry()
np.testing.assert_equal(new_entry.status_code, 201)
# Erase the un-submitted entry and test that the order has been passed correctly
entry_id = new_entry.json()['id']
erase_unsubmitted_entry = requests.delete(
f'{z.zenodo_api_url}/deposit/depositions/{entry_id}',
params=parameters
)
np.testing.assert_equal(erase_unsubmitted_entry.status_code, 204)
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag
)
zenodo.test_upload_to_zenodo()
......@@ -30,8 +30,8 @@ def create_zenodo_metadata(metadata_filename):
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
......@@ -47,12 +47,12 @@ def create_zenodo_metadata(metadata_filename):
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
......@@ -67,13 +67,13 @@ if __name__ == '__main__':
args = parser.parse_args()
z = ZenodoAPI(
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = z.create_new_entry()
new_entry = zenodo.create_new_entry()
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
......@@ -86,7 +86,7 @@ if __name__ == '__main__':
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(
new_upload = zenodo.upload_file_entry(
deposition_id,
name_file=file,
path_file=full_path_file
......@@ -103,7 +103,7 @@ if __name__ == '__main__':
entry_metadata = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(
update_entry = zenodo.update_metadata_entry(
deposition_id,
data=entry_metadata
)
......@@ -115,9 +115,9 @@ if __name__ == '__main__':
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = z.publish_entry(deposition_id)
# publish = zenodo.publish_entry(deposition_id)
# print(publish.json())
print("New deposit correctly published !")
print(f"The new doi should look like 10.5281/{deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
......@@ -30,8 +30,8 @@ def create_zenodo_metadata(metadata_filename):
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
......@@ -47,17 +47,16 @@ def create_zenodo_metadata(metadata_filename):
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
......@@ -73,13 +72,13 @@ if __name__ == '__main__':
args = parser.parse_args()
z = ZenodoAPI(
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - request a new version of an existing deposit
new_version = z.new_version_entry(args.deposit_id)
new_version = zenodo.new_version_entry(args.deposit_id)
if new_version.status_code < 399:
print(f"Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !")
......@@ -91,7 +90,7 @@ if __name__ == '__main__':
# PRE-2 - If you DO NOT want to erase the old files, comment the following lines
old_files_ids = [file['id'] for file in new_version.json()['files']]
for file_id in old_files_ids:
z.erase_file_entry(
zenodo.erase_file_entry(
new_deposition_id,
file_id
)
......@@ -100,7 +99,7 @@ if __name__ == '__main__':
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(
new_upload = zenodo.upload_file_entry(
new_deposition_id,
name_file=file,
path_file=full_path_file
......@@ -116,7 +115,7 @@ if __name__ == '__main__':
update_entry_metadata = json.load(json_file)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(
update_entry = zenodo.update_metadata_entry(
new_deposition_id,
data=update_entry_metadata
)
......@@ -134,4 +133,4 @@ if __name__ == '__main__':
print("New version of the old deposition correctly published !")
print(f"Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}")
print(f"The new doi should look like 10.5281/{new_deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
#!/usr/bin/env python
# E. Garcia
# E. Garcia 2020
# email: garcia 'at' lapp.in2p3.fr
import json
import requests
import numpy as np
from utils_zenodoci import (find_root_directory,
parse_codemeta_and_write_zenodo_metadata_file
)
class ZenodoAPI:
......@@ -12,14 +16,17 @@ class ZenodoAPI:
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
six tasks within the (sandbox.)zenodo api environment:
the following tasks within the (sandbox.)zenodo api environment:
- Create a new deposit,
- Create a new version of an existing deposit,
- Upload files to a specific Zenodo entry,
- Erase (old version) files from an entry (when creating a new_version entry and uploading new_version files),
- Upload information to the entry (Zenodo compulsory deposit information),
- Publish an entry
- Fetches a user's published entries,
- Creates a new deposit,
- Creates a new version of an existing deposit,
- Uploads files to a specific Zenodo entry,
- Erases a non-published entry / new version draft,
- Erases (old version) files from an entry (when creating a new_version entry and uploading
new_version files),
- Uploads information to the entry (Zenodo compulsory deposit information),
- Publishes an entry
:param access_token: str
......@@ -35,13 +42,33 @@ class ZenodoAPI:
self.zenodo_api_url = zenodo_api_url
self.access_token = access_token
self.exist_codemeta_file = False
self.path_codemeta_file = ''
self.exist_zenodo_metadata_file = False
self.path_zenodo_metadata_file = ''
def fetch_user_entries(self):
"""
Fetch the published entries of an user. Works to tests connection to Zenodo too.
GET method to {zenodo_api_url}/deposit/depositions
:return: obj
request.get answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
parameters = {'access_token': self.access_token}
return requests.get(url, params=parameters)
def create_new_entry(self):
"""
Create a new entry / deposition in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
headers = {"Content-Type": "application/json"}
......@@ -53,6 +80,7 @@ class ZenodoAPI:
"""
Upload a file to a Zenodo entry. If first retrieve the entry by a GET method to the
{zenodo_api_url}/deposit/depositions/{entry_id}.
PUT method to {bucket_url}/{filename}. The full api url is recovered when the entry is firstly retrieved.
:param entry_id: str
......@@ -62,7 +90,8 @@ class ZenodoAPI:
:param path_file: str
Path to the file to be uploaded
:return: json requests.put object
:return: obj
json requests.put object
"""
# 1 - Retrieve and recover information of an existing deposit
parameters = {'access_token': self.access_token}
......@@ -78,10 +107,11 @@ class ZenodoAPI:
return upload.json()
def update_info_entry(self, entry_id, data):
def update_metadata_entry(self, entry_id, data):
"""
Update an entry resource. Data should be the entry information that will be shown when a deposition is visited
at the Zenodo site.
PUT method to {zenodo_api_url}/deposit/depositions/{entry_id}. `data` MUST be included as json.dump(data)
:param entry_id: str
......@@ -89,13 +119,33 @@ class ZenodoAPI:
:param data: object
json object containing the metadata (compulsory fields) that are enclosed when a new entry is created.
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
headers = {"Content-Type": "application/json"}
parameters = {'access_token': self.access_token}
return requests.put(url, data=json.dumps(data), headers=headers, params=parameters)
return requests.put(url, data=json.dumps(data),
headers=headers, params=parameters)
def erase_entry(self, entry_id):
"""
Erase an entry/new version of an entry that HAS NOT BEEN published yet.
Any new upload/version will be first saved as 'draft' and not published until confirmation (i.e, requests.post)
DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}.
:param entry_id: str
deposition_id of the Zenodo entry to be erased
:return: obj
requests.delete json answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
parameters = {'access_token': self.access_token}
return requests.delete(url, params=parameters)
def erase_file_entry(self, entry_id, file_id):
"""
......@@ -109,7 +159,8 @@ class ZenodoAPI:
:param file_id: str
Id of the files stored in Zenodo
:return: requests.delete answer
:return: obj
requests.delete answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}"
parameters = {'access_token': self.access_token}
......@@ -124,7 +175,8 @@ class ZenodoAPI:
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.put answer
:return: obj
requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish"
parameters = {'access_token': self.access_token}
......@@ -139,9 +191,120 @@ class ZenodoAPI:
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.post answer
:return: obj
requests.post answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion"
parameters = {'access_token': self.access_token}
return requests.post(url, params=parameters)
def search_codemeta_file(self):
"""Check if a `codemeta.json` files exists in the ROOT directory of the project"""
root_dir = find_root_directory()
codemeta_file = root_dir / 'codemeta.json'
if codemeta_file.exists():
print("\n * Found codemeta.json file within the project !")
self.exist_codemeta_file = True
self.path_codemeta_file = codemeta_file
else:
print("\n ! codemeta.json file NOT found in the root directory of the project !")
def search_zenodo_json_file(self):
"""Check if a `.zenodo.json` files exists in the ROOT directory of the project"""
root_dir = find_root_directory()
zenodo_metadata_file = root_dir / '.zenodo.json'
if zenodo_metadata_file.exists():
print("\n * Found .zenodo.json file within the project !")
self.exist_zenodo_metadata_file = True
self.path_codemeta_file = zenodo_metadata_file
else:
print("\n ! .zenodo.json file NOT found in the root directory of the project !")
def test_upload_to_zenodo(self):
"""
`Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every
stage is the correct one.
Tests:
- If it exists a `codemeta.json` file
- If it exists a `.zenodo.json` file
- If not, it creates one, based on the `codemeta.json` file
- The communication with Zenodo through its API to verify that:
- You can fetch an user entries
- You can create a new entry
- The provided zenodo metadata can be digested, and not errors appear
- Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED !
"""
# Search for codemeta.json file within the project
self.search_codemeta_file()
if self.exist_codemeta_file:
# Search for zenodo.json metadata file within the project
self.search_zenodo_json_file()
if self.exist_zenodo_metadata_file:
print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
else:
print("\n ! Creating a .zenodo.json file from your codemeta.json file...\n"
" Please add, commit and push this file to your project repository.")
self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
self.path_zenodo_metadata_file)
# 1 - Test connection
print("Testing communication with Zenodo...")
test_connection = self.fetch_user_entries()
try:
np.testing.assert_equal(test_connection.status_code, 200)
print("OK !")
except:
print(test_connection.json())
# 2 - Test new entry
print("Testing the creation of a dummy entry to (sandbox)Zenodo...")
new_entry = self.create_new_entry()
try:
np.testing.assert_equal(new_entry.status_code, 201)
print("OK !")
except:
print(new_entry.json())
# 3 - Test upload metadata
print("Testing the ingestion of the Zenodo metadata...")
test_entry_id = new_entry.json()['id']
with open(self.path_zenodo_metadata_file) as file:
metadata_entry = json.load(file)
update_metadata = self.update_metadata_entry(test_entry_id,
data=metadata_entry
)
try:
np.testing.assert_equal(update_metadata.status_code, 200)
print("OK !")
except:
print(update_metadata.json())
# 4 - Test delete entry
print("Testing the deletion of the dummy entry...")
delete_test_entry = self.erase_entry(test_entry_id)
try:
np.testing.assert_equal(delete_test_entry.status_code, 204)
print("OK !")
except:
print(delete_test_entry.json())
print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
"You should not face any trouble when uploading a project to Zenodo through the "
"ESCAPE-GitLabCI pipeline.\n"
"In case you do, please contact us !\n")
else:
print("\n ! Please add a codemeta.json file to the ROOT directory of your project.\n")
......@@ -7,23 +7,34 @@ https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/commits/master)
Library to manage an upload to Zenodo through its REST API.
## _Quickstart_
1. Add a `codementa.json` file to your project.
2. Create a token at (sandbox)zenodo and add it as a variable to your project (see below).
3. **TEST** that you can communicate correctly with Zenodo. You will test and debug at the same time that all the
stages of the GitLabCI-Zenodo pipeline will work correctly.
```bash
$ python .zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
```
## Continuous Deployment to Zenodo
The library was developed specifically to perform a deploy stage (to the Zenodo repository) in a GitLab CI
The library was developed specifically to perform a deploy stage (to the Zenodo repository) within a GitLab CI
pipeline that **could be implemented in any external project**.
It provides a module to handle the upload of the desired
It provides all the needed Scripts to handle the upload of the desired
file(s) to the [ESCAPE2020 Zenodo community](https://zenodo.org/communities/escape2020/).
Please **only upload stable versions/releases of source code and/or image containers!**
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library and
the built Singularity container created in the previous CI stage (check [the ESCAPE project template](
A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the
GitLabCI pipeline.
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library ( as well
as the built Singularity container created in the previous CI stage check [the ESCAPE project template](
https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape)
) to:
- Either upload the desired file(s) to the ESCAPE community in Zenodo.
- Either upload a new version of an existing entry to Zenodo.
A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the GitLabCI pipeline.
- Either upload a new version of an existing entry to Zenodo.
Also, depending on the case, the corresponding python script (`upload_new_deposit.py` or `upload_new_version_of_deposit.py`)
must be adapted and included into the `.gitlab-ci.yml` file with its corresponding arguments (examples are shown in the yml file).
......@@ -67,7 +78,7 @@ your personal token, you should create an environment variable in your GitLab re
The environment variable will look like this:
```sh
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox_zenodo False
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
```
# Contact
......
......@@ -12,8 +12,8 @@
"downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/archive/v1.0/zenodoci-v1.0.tar.gz",
"installUrl": "https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/repository/publish_in_repository/",
"releaseNotes": "Deploy stage (into the Zenodo repository) for a CI pipeline",
"dateCreated": "2020-06-31",
"datePublished": "2020-06-31",
"dateCreated": "2020-06-30",
"datePublished": "2020-06-30",
"dateModified": "2020-10-24",
"isAccessibleForFree": true,
"isPartOf": [
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment