Commit 601100cc authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Merge branch 'allow_test_only_w_zenodo' into 'master'

allow only providing a .zenodo.json file to test the gitlab-zenodo upload

See merge request !11
parents fe0d65a9 ea0cc072
Pipeline #106160 canceled with stages
in 3 minutes and 32 seconds
stages:
- test
- deploy
- test_the_deploy
- test_container
test_install:
stage: test
image: python:3.6.11-buster
image: python:3.6-buster
script:
- apt-get -y update
- pip install .
......@@ -14,7 +14,7 @@ test_install:
deploy_zenodo:
stage: deploy
image: python:3.6.11-buster
image: python:3.6-buster
#dependencies:
### Ideally to be used within a ci pipeline in where a container of the source code is build in a previous stage.
### You can have a look into https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/blob/master/.gitlab-ci.yml
......@@ -26,7 +26,7 @@ deploy_zenodo:
### INFORMATION FOR THE USER; Python, pip and wget are already installed in the container
#- cat /etc/os-release # Debian GNU/Linux 10 (buster)
#- pip3 --version # pip 20.1.1
#- python3 --version # 3.6.11 as expected
#- python3 --version # 3.6 as expected
- pip install requests numpy
- pip install . # Install ZenodoCI
......@@ -35,14 +35,11 @@ deploy_zenodo:
#- test_connection_zenodo -t $ZENODO_TOKEN -s False --project_dir $CI_PROJECT_DIR
script:
### 3 - Get the last tag/release of the repository
- export REPOSITORY_NAME=zenodoci
- export REPOSITORY_BASE_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
### 3 - Get the last tag/release of the repository,
### 4 - Download the repository and move it to the build directory
### If no release is found/correctly parsed, the script will download the last commit pushed to the master branch
- mkdir -p build
- /bin/bash zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL
- /bin/bash zenodoci/parse_last_release.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- ls ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
......@@ -73,7 +70,7 @@ deploy_zenodo:
test_deploy_from_container:
stage: test_the_deploy
stage: test_container
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci
before_script:
- wget -q https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/raw/master/zenodoci/parse_last_release.sh
......
......@@ -3,15 +3,17 @@
# E. Garcia 2020
# email: garcia 'at' lapp.in2p3.fr
import sys
import json
import pprint
import requests
from os.path import abspath
from pathlib import Path
from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
class ZenodoAPI:
def __init__(self, access_token, sandbox=True, proj_root_dir='.'):
def __init__(self, access_token, sandbox=True, proj_root_dir='./'):
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
......@@ -204,7 +206,7 @@ class ZenodoAPI:
# root_dir = find_root_directory()
root_dir = self.proj_root_dir
print(f'Repo root directory {root_dir}') # DEBUG
print(f'\nProject root directory {abspath(root_dir)}') # DEBUG
codemeta_file = root_dir / 'codemeta.json'
......@@ -230,82 +232,98 @@ class ZenodoAPI:
else:
print("\n ! .zenodo.json file NOT found in the root directory of the project !")
def conversion_codemeta2zenodo(self):
"""Perform the codemeta2zenodo conversion if a codemeta.json file is found"""
if self.exist_codemeta_file:
print("\n * Creating a .zenodo.json file from your codemeta.json file...")
self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
self.path_zenodo_metadata_file)
else:
pass
def test_upload_to_zenodo(self):
"""
`Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every
stage is the correct one.
Tests:
- If it exists a `codemeta.json` file
- If it exists a `.zenodo.json` file
- If not, it creates one, based on the `codemeta.json` file
Checks:
- The existence of a `.zenodo.json` file in the ROOT dir of the project
- If not, it checks if it exists a `codemeta.json` file
- If it exists it performs the codemeta2zenodo conversion
- If not, it exits the program
- The communication with Zenodo through its API to verify that:
- You can fetch an user entries
- You can create a new entry
- The provided zenodo metadata can be digested, and not errors appear
- Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED !
"""
# Search for codemeta.json file within the project
# Search for the codemeta.json and the .zenodo.json files within the project
self.search_codemeta_file()
self.search_zenodo_json_file()
if self.exist_codemeta_file:
# Search for zenodo.json metadata file within the project
self.search_zenodo_json_file()
if self.exist_zenodo_metadata_file:
print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
else:
print("\n ! Creating a .zenodo.json file from your codemeta.json file... \n")
self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
self.path_zenodo_metadata_file)
# 1 - Test connection
print("Testing communication with Zenodo...")
test_connection = self.fetch_user_entries()
if test_connection.status_code == 200:
print("Test connection status OK !")
else:
print("ERROR while testing connection status\n", test_connection.json())
if not self.exist_zenodo_metadata_file:
# 2 - Test new entry
print("Testing the creation of a dummy entry to (sandbox)Zenodo...")
new_entry = self.create_new_entry()
if new_entry.status_code == 201:
print("Test new entry status OK !")
if self.exist_codemeta_file:
self.conversion_codemeta2zenodo()
else:
print("ERROR while testing the creation of new entry\n", new_entry.json())
print("\n ! NO codemeta.json NOR .zenodo.json file found. "
"Please add one to the ROOT directory of your project.")
sys.exit(-1)
# 3 - Test upload metadata
print("Testing the ingestion of the Zenodo metadata...")
test_entry_id = new_entry.json()['id']
print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
with open(self.path_zenodo_metadata_file) as file:
metadata_entry = json.load(file)
update_metadata = self.update_metadata_entry(test_entry_id,
data=metadata_entry
)
if update_metadata.status_code == 200:
print("Update metadata status OK !")
pprint.pprint(metadata_entry)
else:
print("ERROR while testing update of metadata\n", update_metadata.json())
# 1 - Test connection
print("1 --> Testing communication with Zenodo...")
# 4 - Test delete entry
print("Testing the deletion of the dummy entry...")
delete_test_entry = self.erase_entry(test_entry_id)
if delete_test_entry.status_code == 204:
print("Delete test entry status OK !")
else:
print("ERROR while deleting test entry\n", delete_test_entry.json())
test_connection = self.fetch_user_entries()
if test_connection.status_code == 200:
print(" * Test connection status OK !")
else:
print(" ! ERROR while testing connection status\n", test_connection.json())
print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
"You should not face any trouble when uploading a project to Zenodo through the "
"ESCAPE-GitLabCI pipeline.\n"
"In case you do, please contact us !\n")
# 2 - Test new entry
print("2 --> Testing the creation of a dummy entry to (sandbox)Zenodo...")
new_entry = self.create_new_entry()
if new_entry.status_code == 201:
print(" * Test new entry status OK !")
else:
print(" ! ERROR while testing the creation of new entry\n", new_entry.json())
# 3 - Test upload metadata
print("3 --> Testing the ingestion of the Zenodo metadata...")
test_entry_id = new_entry.json()['id']
with open(self.path_zenodo_metadata_file) as file:
metadata_entry = json.load(file)
update_metadata = self.update_metadata_entry(test_entry_id,
data=metadata_entry)
if update_metadata.status_code == 200:
print(" * Update metadata status OK !")
pprint.pprint(metadata_entry)
else:
print(" ! ERROR while testing update of metadata\n", update_metadata.json())
print(" ! Erasing dummy test entry...\n")
erase_error = self.erase_entry(test_entry_id)
if erase_error.status_code != 204:
print(f" !! ERROR erasing dummy test entry. Please erase it manually at\n "
f"{self.zenodo_api_url[:-4]}/deposit")
sys.exit(-1)
# 4 - Test delete entry
print("4 --> Testing the deletion of the dummy entry...")
delete_test_entry = self.erase_entry(test_entry_id)
if delete_test_entry.status_code == 204:
print(" * Delete test entry status OK !")
else:
print(" ! ERROR while deleting test entry\n", delete_test_entry.json())
print("\n ! Please add a codemeta.json file to the ROOT directory of your project.\n")
print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
"You should not face any trouble when uploading a project to Zenodo through the "
"ESCAPE-GitLabCI pipeline.\n"
"In case you do, please contact us !\n")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment