Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • garcia/zenodoci
1 result
Show changes
Commits on Source (51)
Hide whitespace changes
Inline Side-by-side
Showing
with 1299 additions and 432 deletions
.gitgignore 0 → 100644
View file @ a1969df0
.pytest_cache
# Compiled files
__pycache__
# Packages/installer info
*.egg
*.eggs
*.egg-info
dist
build
# Mac OSX
.DS_Store
# Pycharm editor project files
.idea
\ No newline at end of file
.gitlab-ci.yml
View file @ a1969df0
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
paths:
- .cache/pip
- venv/
stages: stages:
- install
- test
- build_docker
- deploy - deploy
deploy_zenodo: .virtualenv_template: &virtualenv_definition |
stage: deploy python -V
image: python:3.6.11-buster pip install -U pip setuptools wheel virtualenv
#dependencies: virtualenv venv
### Ideally to be used within a ci pipeline in where a container of the source code is build in a previous stage. source venv/bin/activate
### You can have a look into https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/blob/master/.gitlab-ci.yml pip install pytest pytest-cov
# - build_image pip install -e .
pip freeze
.junit_template: &junit_definition
artifacts:
reports:
junit: "junit*.xml"
install_py36:
stage: install
image: python:3.6-buster
script:
- apt-get -y update
- pip install .
only:
- branches
test_py36:
stage: test
image: python:3.6-buster
script: script:
### 1 - Install dependencies in the image and upload the files to Zenodo - *virtualenv_definition
- apt-get -y update - pytest tests/
--junitxml=junit_py36.xml
### INFORMATION FOR THE USER; Python, pip and wget are already installed in the container --color=yes
#- cat /etc/os-release # Debian GNU/Linux 10 (buster) --verbose
#- pip3 --version # pip 20.1.1 --cov=zenodoci
#- python3 --version # 3.6.11 as th --cov-report=xml
- pip3 install requests --cov-report=term
<<: *junit_definition
### 2 - Get the last tag/release of the repository
- export REPOSITORY_NAME=zenodoci
- export REPOSITORY_BASE_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
- export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'`
### 3 - Download the repository and move it to the build directory
### If no release is found/correctly parsed, the script will download the last commit pushed to the master branch
- mkdir -p build
- >
if [ -z "$LAST_RELEASE" ]; then
echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"; \
wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip; \
mv $REPOSITORY_NAME-master.zip ./build
else
echo "$LAST_RELEASE tag / release found !"; \
wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip; \
mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
fi
- ls ./build
### 4 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
- >
python3 .zenodoci/upload_new_deposit.py
--token $SANDBOX_ZENODO_TOKEN
--sandbox_zenodo True
--input-directory ./build
### 4 - To deploy a NEW DEPOSIT to ZENODO
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $ZENODO_TOKEN
# --sandbox_zenodo False
# --input-directory ./build
### 4 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
--token $SANDBOX_ZENODO_TOKEN
--sandbox_zenodo True
--input-directory ./build
--deposit_id $DEPOSIT_ID_ZENODOCI
# --token $ZENODO_TOKEN
# --sandbox_zenodo False
only: only:
### Ideally this stage should be run only when a new release / tag of the source code is created (- tags). - branches
# The script is changed to check that the both `upload_new_deposit` and `upload_new_version_deposit` works nicely.
- tags build_docker:
- master stage: build_docker
\ No newline at end of file image: docker:19.03.12
services:
- docker:19.03.12-dind
before_script:
- cat /etc/os-release # "Alpine Linux v3.12"
- apk add git
- export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'`
- echo $LAST_RELEASE
script:
- cd Docker
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE .
- docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE
only:
- tags
deploy_sandbox:
stage: deploy
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci:v1.2
before_script:
- which parse_last_release_git.sh
- test_connection_zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR
script:
- pwd # testing
- ls # testing
- env # testing
- upload_new_deposit -h
- upload_new_version_deposit -h
- codemeta2zenodo -h
- mkdir -p build
- parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
- ls ./build
- upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build
- upload_new_version_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build --deposit_id $SANDBOX_ZENODO_PROJECT_ID
except:
- master # run only in branches
- tags
deploy_zenodo:
stage: deploy
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci:v1.2
before_script:
- test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
script:
- mkdir -p build
- parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
- ls ./build
- upload_new_version_deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
only:
- tags
.zenodoci/__init__.py deleted 100644 → 0
View file @ 9a5c6057
.zenodoci/repository_information.json deleted 100644 → 0
View file @ 9a5c6057
{
"metadata": {
"title": "ZenodoCI",
"upload_type": "software",
"description": "The library is intended to be part of a complete CI pipeline. This stage deploys to Zenodo the files found in the ./build directory, configured in the .gitlab-ci.yml file",
"creators": [{"name": "Garcia, Enrique",
"affiliation": "LAPP, CNRS",
"orcid": "0000-0003-2224-4594"}],
"access_right": "open",
"license": "MIT",
"communities": [{"identifier": "escape2020"}],
"keywords": [],
"language": "eng",
"notes": "Deploy stage (into the Zenodo repository) for a CI (GitLab) pipeline.",
"version": "1.0",
"grants": [{"id": "10.13039/501100000780::824064"}]
}
}
\ No newline at end of file
.zenodoci/upload_new_deposit.py deleted 100644 → 0
View file @ 9a5c6057
# -*- coding: utf-8 -*-
import os
import json
import argparse
from distutils.util import strtobool
from .zenodoapi import ZenodoAPI
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
args = parser.parse_args()
if __name__ == '__main__':
z = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = z.create_new_entry()
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
doi = new_entry.json()['metadata']['prereserve_doi']['doi']
print(f"Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
else:
print(new_entry.json())
# 2 - upload files
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(deposition_id,
name_file=file,
path_file=full_path_file)
print(f"File {file} correctly uploaded !\n", new_upload)
# 3 - Upload repository information - that you must have filled before the json file !
with open('.zenodoci/repository_information.json') as json_file:
entry_info = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(deposition_id,
data=entry_info)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !")
else:
print(f"Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = z.publish_entry(deposition_id)
# print(publish.json())
print("New deposit correctly published !")
print(f"The new doi should look like 10.5281/{deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}deposit/{deposition_id} **")
.zenodoci/upload_new_version_deposit.py deleted 100644 → 0
View file @ 9a5c6057
# -*- coding: utf-8 -*-
import os
import json
import argparse
from distutils.util import strtobool
from .zenodoapi import ZenodoAPI
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
parser.add_argument('--deposit_id', '-id', type=str,
dest='deposit_id',
help='deposit_id of the deposit that is going to be updated by a new version',
required=True)
args = parser.parse_args()
if __name__ == '__main__':
z = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - request a new version of an existing deposit
new_version = z.new_version_entry(args.deposit_id)
if new_version.status_code < 399:
print(f"Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !")
else:
print(new_version.json())
new_deposition_id = new_version.json()['links']['latest_draft'].rsplit('/')[-1]
# 2-PRE If you DO NOT want to erase the old files, comment the following lines
old_files_ids = [file['id'] for file in new_version.json()['files']]
for file_id in old_files_ids:
z.erase_file_entry(new_deposition_id,
file_id)
# 2 - Upload new version of file(s)
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(new_deposition_id,
name_file=file,
path_file=full_path_file)
print(f"File {file} correctly uploaded !\n", new_upload)
# 3 - Update metadata info
with open('.zenodoci/repository_information.json') as json_file:
update_entry_info = json.load(json_file)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(new_deposition_id,
data=update_entry_info)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !\n")
else:
print(f"Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = z.publish_entry(new_deposition_id)
# print(publish.json())
print("New version of the old deposition correctly published !")
print(f"Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}")
print(f"The new doi should look like 10.5281/{new_deposition_id}. However please")
print(f" ** Check the upload at {z.zenodo_api_url[:-4]}deposit/{new_deposition_id} **")
.zenodoci/zenodoapi.py deleted 100644 → 0
View file @ 9a5c6057
# -*- coding: utf-8 -*-
import json
import requests
class ZenodoAPI:
def __init__(self, access_token, sandbox=True):
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
six tasks within the (sandbox.)zenodo api environment:
- Create a new deposit,
- Create a new version of an existing deposit,
- Upload files to a specific Zenodo entry,
- Erase (old version) files from an entry (when creating a new_version entry and uploading new_version files),
- Upload information to the entry (Zenodo compulsory deposit information),
- Publish an entry
:param access_token: str
Personal access token to (sandbox.)zenodo.org/api
:param sandbox: bool
Communicates with either zenodo or sandbox.zenodo api
"""
if sandbox:
zenodo_api_url = "https://sandbox.zenodo.org/api"
else:
zenodo_api_url = "https://zenodo.org/api"
self.zenodo_api_url = zenodo_api_url
self.access_token = access_token
def create_new_entry(self):
"""
Create a new entry / deposition in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions
:return: requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
headers = {"Content-Type": "application/json"}
parameters = {'access_token': self.access_token}
return requests.post(url, json={}, headers=headers, params=parameters)
def upload_file_entry(self, entry_id, name_file, path_file):
"""
Upload a file to a Zenodo entry. If first retrieve the entry by a GET method to the
{zenodo_api_url}/deposit/depositions/{entry_id}.
PUT method to {bucket_url}/{filename}. The full api url is recovered when the entry is firstly retrieved.
:param entry_id: str
deposition_id of the Zenodo entry
:param name_file: str
File name of the file when uploaded
:param path_file: str
Path to the file to be uploaded
:return: json requests.put object
"""
# 1 - Retrieve and recover information of an existing deposit
parameters = {'access_token': self.access_token}
fetch = requests.get(f"{self.zenodo_api_url}/deposit/depositions/{entry_id}",
params=parameters)
# 2 - Upload the files
bucket_url = fetch.json()['links']['bucket'] # full url is recovered from fetch (GET) method
url = f"{bucket_url}/{name_file}"
with open(path_file, 'rb') as upload_file:
upload = requests.put(url, data=upload_file, params=parameters)
return upload.json()
def update_info_entry(self, entry_id, data):
"""
Update an entry resource. Data should be the entry information that will be shown when a deposition is visited
at the Zenodo site.
PUT method to {zenodo_api_url}/deposit/depositions/{entry_id}. `data` MUST be included as json.dump(data)
:param entry_id: str
deposition_id of the Zenodo entry
:param data: object
json object containing the metadata (compulsory fields) that are enclosed when a new entry is created.
:return: requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
headers = {"Content-Type": "application/json"}
parameters = {'access_token': self.access_token}
return requests.put(url, data=json.dumps(data), headers=headers, params=parameters)
def erase_file_entry(self, entry_id, file_id):
"""
Erase a file from an entry resource.
This method is intended to be used for substitution of files (deletion) within an entry by their correspondent
new versions.
DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}
:param entry_id: str
deposition_id of the Zenodo entry
:param file_id: str
Id of the files stored in Zenodo
:return: requests.delete answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}"
parameters = {'access_token': self.access_token}
return requests.delete(url, params=parameters)
def publish_entry(self, entry_id):
"""
Publishes an entry in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.put answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish"
parameters = {'access_token': self.access_token}
return requests.post(url, params=parameters)
def new_version_entry(self, entry_id):
"""
Creates a new version of AN EXISTING entry resource.
POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.post answer
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion"
parameters = {'access_token': self.access_token}
return requests.post(url, params=parameters)
Docker/Dockerfile 0 → 100644
View file @ a1969df0
FROM continuumio/miniconda3
ENV PATH=/opt/conda/bin/:${PATH}
SHELL [ "/bin/bash", "-c"]
RUN conda install git pip && \
pip --no-cache-dir install git+https://gitlab.in2p3.fr/escape2020/wp3/zenodoci
README.md
View file @ a1969df0
# This project is now part of the [eossr library](https://gitlab.in2p3.fr/escape2020/wp3/)
# ZenodoCI # ZenodoCI
[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4786641.svg)](https://doi.org/10.5281/zenodo.4786641)
[![pipeline status](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/badges/master/pipeline.svg)]( [![pipeline status](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/badges/master/pipeline.svg)](
https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/commits/master) https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/commits/master)
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
[![coverage report](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/badges/master/coverage.svg)](
https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/commits/master)
Library to manage an upload to Zenodo through its REST API. Library to manage an upload to Zenodo through its REST API.
## Install
```sh
$ git clone https://gitlab.in2p3.fr/escape2020/wp3/zenodoci.git
$ cd zenodoci
$ pip install .
```
Then you would be able to run any of the following;
```sh
$ upload_new_deposit --token $ZENODO_TOKEN --sandbox False --input_dir <./path_proj_dir>
```
```sh
$ upload_new_version_deposit -t $ZENODO_TOKEN -s False -i <./your_proj_dir> -id <ZENODO_PROJ_ID>
```
```sh
$ test_connection_zenodo -t $ZENODO_TOKEN -s False --repo_dir <PROJ_DIR>
```
-----------
Please note that all the code developed in this project, together with the [codemeta2zenodo crosswalk](
https://gitlab.in2p3.fr/escape2020/wp3/codemeta2zenodo) are included and installed into a Docker container that can be
found at [the container registry](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/container_registry) of this project.
**Fell free to `pull` it and use at any moment**.
## _Quickstart_ - Deploy your project to Zenodo (`one-click-build-and-publish`).
1. Include a `codementa.json` metadata file to your project.
- [CodeMeta generator](https://codemeta.github.io/codemeta-generator/).
- The ZenodoCI pipeline will not work without one.
2. Create a token at [zenodo](https://zenodo.org/) and add it as an environment variable to your project (see
[below](#zenodo-token-gitlab-ci-environment-variable)).
3. Add in your `.gitlab-ci.yml` file the following stage to deploy your project into Zenodo.
4. Create a release.
The code to be added to the `.gitlab-ci.yml` file is the following (**COMMON FOR EVERY PROJECT**).
### To create a new entry or deposit
```
stages:
- `(...) all your CI stages (...)`
- deploy
`(...) All the code to be run in your CI pipeline (...)`
deploy_to_zenodo:
stage: deploy
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci:v1.2
before_script:
- test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
script:
- mkdir -p build
- parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
- ls ./build
- upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build
only:
- tags
```
## Continuous Deployment to Zenodo ### To create a new version of an existing entry
Substitute
```
- upload_new_version_deposit -t $ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
```
for
```
- upload_new_version_deposit -t $ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
```
and please check the correct indentation on the file.
The library was developed specifically to perform a deploy stage (to the Zenodo repository) in a GitLab CI ## Zenodo token & GitLab CI environment variable
To allow GitLab communicate with Zenodo through their APIs, a personal access token must be created and included into
the GitLab project (as a **masked !** environment variable).
Please note that (to date) the token can be assigned **only to a single** Zenodo account. To create the token:
- Go to [zenodo.org](https://zenodo.org/).
- Click on `Account` --> `Applications` --> `Personal access token` --> `New token` and Select the desired `Scopes`.
This token will be passed later in the deployment stage of the ZenodoCI pipeline (the same that you should also include
into your project to automatise the upload).
For not sharing publicly your personal token, you should create an environment variable in your GitLab repository.
This way, the token could be used as a variable without revealing its value. To create an an environment variable:
- Go to your GitLab project.
- Click on `Settings` --> `CI/CD` --> `Variables` --> `Add variable` --> Fill the fields --> **Mask your variable(s) !!**
Please name your environment variables as follows (so that no changes should be done to the `.gitlab-ci.yml` file);
- `ZENODO_TOKEN` or `SANDBOX_ZENODO_TOKEN`
- `ZENODO_PROJECT_ID`
so that environment variable(s) should look like:
```sh
$ upload_new_deposit --token $ZENODO_TOKEN --sandbox False --input-directory <./build>
$ upload_new_version_deposit -t $ZENODO_TOKEN -s False -i <./your_proj_dir> -id $ZENODO_PROJECT_ID
```
# Contact
[Contact us](mailto:garcia@lapp.in2p3.fr) for any question or doubt related with the current library, the ESCAPE project,
the metadata schema context or the integration of any project to Zenodo, Gitlab or the ESCAPE OSSR.
<!-- COMMENT
## Continuous Integration to Zenodo
The library was developed specifically to perform a deploy stage (to the Zenodo repository) within a GitLab CI
pipeline that **could be implemented in any external project**. pipeline that **could be implemented in any external project**.
It provides a module to handle the upload of
specified file(s) to the [ESCAPE2020 Zenodo community](https://zenodo.org/communities/escape2020/). It provides all the needed Scripts to handle the upload of the desired
file(s) to the [ESCAPE2020 Zenodo community](https://zenodo.org/communities/escape2020/).
Please **only upload stable versions/releases of source code and/or image containers!** Please **only upload stable versions/releases of source code and/or image containers!**
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library and A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the
the built Singularity container created in the previous CI stage (check [the ESCAPE project template]( GitLabCI pipeline.
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library ( as well
as the built Singularity container created in the previous CI stage check [the ESCAPE project template](
https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape) https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape)
) to: ) to:
- Either upload the desired file(s) to the ESCAPE community in Zenodo. - Either upload the desired file(s) to the ESCAPE community in Zenodo.
- Either upload a new version of an existing entry to Zenodo. - Either upload a new version of an existing entry to Zenodo.
The `repository_information.json` file must be filled up before pushing to the GitLab repository. This file will be Also, depending on the case, the corresponding python script (`upload_new_deposit.py` or `upload_new_version_of_deposit.py`)
used to fill up the compulsory information that a Zenodo entry must contain. Also, depending on the case, the must be adapted and included into the `.gitlab-ci.yml` file with its corresponding arguments (examples are shown in the yml file).
corresponding python script (`upload_new_deposit.py` or `upload_new_version_of_deposit.py`) must be adapted and
included into the `.gitlab-ci.yml` file with its corresponding arguments (examples are shown in the yml file). - **You need a more detailed tutorial ?** Please check our more detailed tutorials at the [**OSSR entry point site**](
https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/repository/publish_in_repository/).
### Zenodo token & GitLab CI environment variable
- **Problems creating a `.gitlab-ci.yml` file ?** Please check the `gitlab_generator` module implemented in [**this ESCAPE project**](
To connect the GitLab repository with Zenodo in an autonomous way, a personal access token must be created. This token https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/tree/master/gitlabci_generator).
is assigned to a **single** Zenodo account, and it will allow the interaction with
(sandbox.)zenodo through its API. To create the token:
- Go to (sandbox)zenodo.org
- Account --> Applications --> Personal access token --> New token.
This token will be passed later in the deployment stage of the CI pipeline. For not sharing publicly #### **Use of new metadata context ! Please check the news !**
your personal token, you should create an environment variable in your GitLab repository. This way, the token could be
used as a variable without revealing its value. To create an an environment variable:
- Go to your GitLab repository.
- Settings --> CI/CD --> Variables --> Add variable --> Fill the fields --> Mask your variable(s) !!
The environment variable will look like this: We are no longer supporting the use of a `repository_information.json` file to provide metadata to Zenodo.
```sh We are currently moving to a [CodeMeta metadata context](https://codemeta.github.io/).
$ python .zenodoci/upload_new_deposit.py -i build -t $ZENODO_TOKEN -s False This metadata standard provides a complete metadata schema context supported by many other services and search engines.
```
\ No newline at end of file Adding a single `codemeta.json` file to the root directory of your project will be enough ! Please check out the
[ESCAPE metadata template](https://gitlab.in2p3.fr/escape2020/wp3/escape_metadata_template) project for a _quickstart_ on
how to easily create a `codemeta.json` file.
Last but not least ! Please note that during the CI pipeline the `zenodoci` module will search for a `codemeta.json` file
and will automatically create the equivalent file to provide metadata to the Zenodo repository. The `.zenodo.json` file
will contain the exactly same information that in the `codemeta.json` file but using the Zenodo syntax.
-->
codemeta.json 0 → 100644
View file @ a1969df0
{
"@context": "https://doi.org/10.5063/schema/codemeta-2.0",
"@type": "SoftwareSourceCode",
"name": "ZenodoCI",
"description": "The library is intended to be part of a complete CI pipeline. This stage deploys to Zenodo the files found in the ./build directory, configured in the .gitlab-ci.yml file",
"keywords": ["ESCAPE"],
"license": "https://spdx.org/licenses/MIT",
"softwareVersion": "1.2",
"developmentStatus": "active",
"codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci",
"runtimePlatform": "Python >3.6",
"downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/archive/v1.0/zenodoci-v1.0.tar.gz",
"installUrl": "https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/repository/publish_in_repository/",
"releaseNotes": "Deploy stage (into the Zenodo repository) for a CI pipeline",
"dateCreated": "2020-06-30",
"datePublished": "2020-06-30",
"dateModified": "2021-05-25",
"isAccessibleForFree": true,
"isPartOf": [
"https://gitlab.in2p3.fr/escape2020",
"https://gitlab.in2p3.fr/escape2020/wp3",
"https://zenodo.org/communities/escape2020",
"https://projectescape.eu/"
],
"contIntegration": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/pipelines",
"buildInstructions": "https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/repository/publish_in_repository/",
"issueTracker": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/issues",
"readme": "https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/blob/master/README.md",
"programmingLanguage": [
{
"@type": "ComputerLanguage",
"name": "Python",
"url": "https://www.python.org/"
},
{
"@type": "ComputerLanguage",
"name": "Bash",
"url": "https://www.gnu.org/software/bash/"
}
],
"softwareRequirements": [
{
"@type": "SoftwareApplication",
"identifier": "requests",
"name": "requests",
"softwareVersion": ">=2.23"
},
{
"@type": "SoftwareApplication",
"identifier": "pyjson",
"name": "json",
"softwareVersion": ">=1.3.0"
}
],
"maintainer": {
"@type": "Person",
"@id": "https://orcid.org/0000-0003-2224-4594",
"givenName": "Enrique",
"familyName": "Garcia",
"email": "garcia@lapp.in2p3.fr",
"affiliation": {
"@type": "Organization",
"name": "LAPP, CNRS"
}
},
"author": [
{
"@type": "Person",
"@id": "https://orcid.org/0000-0003-2224-4594",
"givenName": "Enrique",
"familyName": "Garcia",
"email": "garcia@lapp.in2p3.fr",
"affiliation": {
"@type": "Organization",
"name": "LAPP, CNRS"
}
}
],
"contributor": [
{
"@type": "Person",
"@id": "https://orcid.org/0000-0002-5686-2078",
"givenName": "Thomas",
"familyName": "Vuillaume",
"email": "vuillaume@lapp.in2p3.fr",
"affiliation": {
"@type": "Organization",
"name": "LAPP, CNRS"
}
}
],
"funder":[
{
"@type": "Organization",
"name": "ESCAPE: European Science Cluster of Astronomy & Particle physics ESFRI research infrastructures",
"funder": {
"@type": "Organization",
"@id": "https://doi.org/10.13039/501100000780",
"name": "European Commission"
}
}
],
"funding": "824064"
}
\ No newline at end of file
setup.py
View file @ a1969df0
#!/usr/bin/env python #!/usr/bin/env python
import setuptools import re
from setuptools import setup, find_packages
setuptools.setup( entry_points = {'console_scripts': [
'upload_new_deposit = zenodoci.upload_new_deposit:main',
'upload_new_version_deposit = zenodoci.upload_new_version_deposit:main',
'test_connection_zenodo = zenodoci.test_connection_zenodo:main'
]
}
def get_property(prop, project):
result = re.search(r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop), open(project + '/__init__.py').read())
return result.group(1)
setup(
name='zenodoci', name='zenodoci',
version='1.0', version=get_property('__version__', 'zenodoci'),
author='Garcia, Enrique', description="REST API handler to manage the uploads and communication with Zenodo.",
install_requires=[
'requests',
'codemeta2zenodo @ git+https://gitlab.in2p3.fr/escape2020/wp3/codemeta2zenodo',
],
packages=find_packages(),
scripts=['zenodoci/parse_last_release_git.sh'],
# tests_require=['pytest'],
author='Enrique Garcia',
author_email='garcia<at>lapp.in2p3.fr', author_email='garcia<at>lapp.in2p3.fr',
description="Library to manage an upload to Zenodo through its REST API.",
packages=setuptools.find_packages(),
install_requires=['numpy', 'requests'],
license='Open Source. MIT license. See LICENSE file.',
url='https://gitlab.in2p3.fr/escape2020/wp3/zenodoci', url='https://gitlab.in2p3.fr/escape2020/wp3/zenodoci',
license='MIT',
entry_points=entry_points
) )
tests/test_zenodoapi.py 0 → 100644
View file @ a1969df0
#!/usr/bin/env python
import json
import unittest
import requests
from pathlib import Path
from os import remove
from os.path import dirname, realpath, join
from zenodoci.zenodoapi import ZenodoAPI
ROOT_DIR = dirname(dirname(realpath(__file__)))
class TestZenodoApiSandbox(unittest.TestCase):
def test_initialization_sandbox(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=True,
proj_root_dir=ROOT_DIR)
assert isinstance(z, ZenodoAPI)
assert z.zenodo_api_url == 'https://sandbox.zenodo.org/api'
assert z.access_token == token
assert type(z.exist_codemeta_file) == bool
assert type(z.exist_zenodo_metadata_file) == bool
assert z.proj_root_dir == Path(ROOT_DIR)
assert z.path_codemeta_file == z.proj_root_dir
assert z.path_zenodo_metadata_file == z.proj_root_dir
assert isinstance(z.proj_root_dir, (Path, str))
class TestZenodoAPI(unittest.TestCase):
def test_initialization(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=False,
proj_root_dir=ROOT_DIR)
assert isinstance(z, ZenodoAPI)
assert z.zenodo_api_url == 'https://zenodo.org/api'
assert z.access_token == token
assert type(z.exist_codemeta_file) == bool
assert type(z.exist_zenodo_metadata_file) == bool
assert z.proj_root_dir == Path(ROOT_DIR)
assert z.path_codemeta_file == z.proj_root_dir
assert z.path_zenodo_metadata_file == z.proj_root_dir
assert isinstance(z.proj_root_dir, (Path, str))
def test_zenodo_api_methods(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=False,
proj_root_dir=ROOT_DIR)
test_id = '42'
z.search_codemeta_file()
test_filename = join(ROOT_DIR, z.path_codemeta_file)
path_test_filename = './'
fetch_user_entry = z.fetch_user_entries()
create_new_entry = z.create_new_entry()
fetch_single_entry = z.fetch_entry(
entry_id=test_id
)
# upload_file_entry = z.upload_file_entry(
# entry_id=test_id,
# name_file=test_filename,
# path_file=path_test_filename
# )
upload_metadata_entry = z.update_metadata_entry(
entry_id=test_id,
json_metadata=test_filename
)
erase_entry = z.erase_entry(
entry_id=test_id
)
erase_file_entry = z.erase_file_entry(
entry_id=test_id,
file_id=test_id
)
publish_entry = z.publish_entry(
entry_id=test_id
)
new_version_entry = z.new_version_entry(
entry_id=test_id
)
community_entries = z.fetch_community_entries()
fetch_ids = z.fetch_community_entries_per_id()
fetch_filenames = z.fetch_community_entries_per_title()
assert isinstance(fetch_user_entry, requests.models.Response)
assert isinstance(create_new_entry, requests.models.Response)
assert isinstance(fetch_single_entry, requests.models.Response)
# assert isinstance(upload_file_entry, requests.models.Response)
assert isinstance(upload_metadata_entry, requests.models.Response)
assert isinstance(erase_entry, requests.models.Response)
assert isinstance(erase_file_entry, requests.models.Response)
assert isinstance(publish_entry, requests.models.Response)
assert isinstance(new_version_entry, requests.models.Response)
assert isinstance(community_entries, requests.models.Response)
assert isinstance(fetch_ids, list)
assert isinstance(fetch_filenames, list)
def test_search_codemeta_file(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=False,
proj_root_dir=ROOT_DIR)
assert z.exist_codemeta_file is False
z.search_codemeta_file()
assert z.exist_codemeta_file is True
codemeta_file_path = Path(ROOT_DIR) / 'codemeta.json'
assert z.path_codemeta_file == codemeta_file_path
assert codemeta_file_path.is_file()
print(z.path_codemeta_file, type(z.path_codemeta_file))
with open(z.path_codemeta_file) as f:
json.load(f)
def test_search_zenodo_json_file(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=False,
proj_root_dir=ROOT_DIR)
assert z.exist_zenodo_metadata_file is False
z.search_zenodo_json_file()
assert z.exist_zenodo_metadata_file is False
def test_conversion_codemeta2zenodo_and_search_zenodo_file(self):
token = 'FakeToken'
z = ZenodoAPI(access_token=token,
sandbox=False,
proj_root_dir=ROOT_DIR)
z.search_codemeta_file()
z.conversion_codemeta2zenodo()
z.search_zenodo_json_file()
assert z.exist_zenodo_metadata_file is True
zenodo_file_path = Path(ROOT_DIR) / '.zenodo.json'
assert z.path_zenodo_metadata_file == zenodo_file_path
assert zenodo_file_path.is_file()
with open(z.path_zenodo_metadata_file) as f:
json.load(f)
remove(z.path_zenodo_metadata_file)
zenodoci/__init__.py 0 → 100644
View file @ a1969df0
__version__ = "1.1"
zenodoci/parse_last_release_git.sh 0 → 100755
View file @ a1969df0
#!/usr/bin/env bash
REPOSITORY_NAME="$1"
REPOSITORY_BASE_URL="$2"
LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'`
if [ -z "$LAST_RELEASE" ]; then
echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"
wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip
mv $REPOSITORY_NAME-master.zip ./build
else
echo "$LAST_RELEASE tag / release found !"
wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip
mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
fi
zenodoci/test_connection_zenodo.py 0 → 100644
View file @ a1969df0
#!/usr/bin/env python
# E. Garcia Nov 2020
# Module to test the connection and the upload of new entries/version to Zenodo.
import argparse
from distutils.util import strtobool
from zenodoci.zenodoapi import ZenodoAPI
def main():
# Required arguments
parser = argparse.ArgumentParser(description="Test the connection to zenodo and all the stages of a new upload.")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--project_dir', '-p', action='store',
dest='project_dir',
help='Path to the root directory of the directory to be uploaded. '
'DEFAULT; assumed to be on it, i.e., "./"',
default='./'
)
args = parser.parse_args()
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag,
proj_root_dir=args.project_dir
)
zenodo.test_upload_to_zenodo()
if __name__ == '__main__':
main()
zenodoci/upload_new_deposit.py 0 → 100644
View file @ a1969df0
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
from pathlib import Path
from distutils.util import strtobool
from zenodoci.zenodoapi import ZenodoAPI
from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
def create_zenodo_metadata(metadata_filename, repo_root_dir='./'):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
param metadata_filename: str
path and name to the zenodo metadata json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
# root_dir = find_root_directory()
root_dir = Path(repo_root_dir)
print(f'working dir : {root_dir}') # DEBUG
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(f'JSON files found {files_json}')
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
def main():
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-dir', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
args = parser.parse_args()
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = zenodo.create_new_entry()
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
doi = new_entry.json()['metadata']['prereserve_doi']['doi']
print(f" * Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
else:
print(f" ! ERROR; the NEW entry COULD NOT be created.")
print(new_entry.json())
# 2 - upload files
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = zenodo.upload_file_entry(
deposition_id,
name_file=file,
path_file=full_path_file
)
print(f" * File {file} correctly uploaded !\n", new_upload)
# 3 - Create the zenodo metadata file from a codemeta.json file
zenodo_metadata_filename = '.zenodo.json'
create_zenodo_metadata(zenodo_metadata_filename)
# And upload the repository metadata
with open(zenodo_metadata_filename) as json_file:
entry_metadata = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = zenodo.update_metadata_entry(
deposition_id,
json_metadata=entry_metadata
)
if update_entry.status_code < 399:
print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !")
else:
print(f" ! Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = zenodo.publish_entry(deposition_id)
# print(publish.json())
print(" * New deposit correctly published !\n")
print(f"The new doi should look like 10.5281/{deposition_id}. However please")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
if __name__ == '__main__':
main()
zenodoci/upload_new_version_deposit.py 0 → 100644
View file @ a1969df0
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
from pathlib import Path
from distutils.util import strtobool
from zenodoci.zenodoapi import ZenodoAPI
from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
def create_zenodo_metadata(metadata_filename, repo_root_dir='./'):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
param metadata_filename: str
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
# root_dir = find_root_directory()
root_dir = Path(repo_root_dir)
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(f'JSON files : {files_json}')
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
def main():
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-dir', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
parser.add_argument('--deposit_id', '-id', type=str,
dest='deposit_id',
help='deposit_id of the deposit that is going to be updated by a new version',
required=True)
args = parser.parse_args()
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - request a new version of an existing deposit
new_version = zenodo.new_version_entry(args.deposit_id)
if new_version.status_code < 399:
print(f" * Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !")
else:
print(f" ! ERROR; new version of the {args.deposit_id} entry COULD NOT be created.")
print(new_version.json())
new_deposition_id = new_version.json()['links']['latest_draft'].rsplit('/')[-1]
# PRE-2 - If you DO NOT want to erase the old files, comment the following lines
old_files_ids = [file['id'] for file in new_version.json()['files']]
for file_id in old_files_ids:
zenodo.erase_file_entry(
new_deposition_id,
file_id
)
# 2 - Upload new version of file(s)
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = zenodo.upload_file_entry(
new_deposition_id,
name_file=file,
path_file=full_path_file
)
print(f" * File {file} correctly uploaded !\n", new_upload)
# 3 - Look for a zenodo metadata file, otherwise try to create one
zenodo_metadata_filename = '.zenodo.json'
create_zenodo_metadata(zenodo_metadata_filename)
with open(zenodo_metadata_filename) as json_file:
update_entry_metadata = json.load(json_file)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = zenodo.update_metadata_entry(
new_deposition_id,
json_metadata=update_entry_metadata
)
if update_entry.status_code < 399:
print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !\n")
else:
print(f" ! Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = zenodo.publish_entry(new_deposition_id)
# print(publish.json())
print(" * New version of the old deposition correctly published !\n")
print(f"Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}")
print(f"The new doi should look like 10.5281/{new_deposition_id}. However please")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
if __name__ == '__main__':
main()
zenodoci/zenodoapi.py 0 → 100644
View file @ a1969df0
#!/usr/bin/env python
# E. Garcia 2020
# email: garcia 'at' lapp.in2p3.fr
import sys
import json
import pprint
import requests
from os.path import abspath
from pathlib import Path
from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
class ZenodoAPI:
def __init__(self, access_token, sandbox=True, proj_root_dir='./'):
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
the following tasks within the (sandbox.)zenodo api environment:
- Fetches a user's published entries,
- Creates a new deposit,
- Fetches any published record,
- Creates a new version of an existing deposit,
- Uploads files to a specific Zenodo entry,
- Erases a non-published entry / new version draft,
- Erases (old version) files from an entry (when creating a new_version entry and uploading
new_version files),
- Uploads information to the entry (Zenodo compulsory deposit information),
- Publishes an entry
- Find all the published community entries
* per title
* per entry_id
:param access_token: str
Personal access token to (sandbox.)zenodo.org/api
:param sandbox: bool
Communicates with either zenodo or sandbox.zenodo api
"""
if sandbox:
zenodo_api_url = "https://sandbox.zenodo.org/api"
else:
zenodo_api_url = "https://zenodo.org/api"
self.zenodo_api_url = zenodo_api_url
self.access_token = access_token
self.parameters = {'access_token': self.access_token}
self.proj_root_dir = Path(proj_root_dir)
self.exist_codemeta_file = False
self.path_codemeta_file = self.proj_root_dir
self.exist_zenodo_metadata_file = False
self.path_zenodo_metadata_file = self.proj_root_dir
def fetch_user_entries(self):
"""
Fetch the published entries of an user. Works to tests connection to Zenodo too.
GET method to {zenodo_api_url}/deposit/depositions
:return: request.get method
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
return requests.get(url, params=self.parameters)
def create_new_entry(self):
"""
Create a new entry / deposition in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions
:return: request.put method
"""
url = f"{self.zenodo_api_url}/deposit/depositions"
headers = {"Content-Type": "application/json"}
return requests.post(url, json={}, headers=headers, params=self.parameters)
def fetch_entry(self, entry_id):
"""
Fetches (recovers all the existing information, as well as links) of an existing Zenodo entry.
GET method to {zenodo_api_url}/deposit/depositions/{entry_id}
:param entry_id: str
entry_id of the entry to fetch
:return: request.get method
"""
# In case of entries created by oneself, usually in the process of being created, the method to fetch
# a record is request.get('api/deposit/deposition/{entry_id}'), see the upload_file_entry methods
# To fetch any other entry, already published, use
url = f"{self.zenodo_api_url}/records/{entry_id}"
return requests.get(url, params=self.parameters)
def upload_file_entry(self, entry_id, name_file, path_file):
"""
Upload a file to a Zenodo entry. If first retrieve the entry by a GET method to the
{zenodo_api_url}/deposit/depositions/{entry_id}.
PUT method to {bucket_url}/{filename}. The full api url is recovered when the entry is firstly retrieved.
:param entry_id: str
deposition_id of the Zenodo entry
:param name_file: str
File name of the file when uploaded
:param path_file: str
Path to the file to be uploaded
:return: request.put method
"""
# 1 - Retrieve and recover information of a record that is in process of being published
fetch = requests.get(f"{self.zenodo_api_url}/deposit/depositions/{entry_id}",
params=self.parameters)
# 2 - Upload the files
bucket_url = fetch.json()['links']['bucket'] # full url is recovered from previous GET method
url = f"{bucket_url}/{name_file}"
with open(path_file, 'rb') as upload_file:
upload = requests.put(url, data=upload_file, params=self.parameters)
return upload.json()
def update_metadata_entry(self, entry_id, json_metadata):
"""
Update an entry resource. Data should be the entry information that will be shown when a deposition is visited
at the Zenodo site.
PUT method to {zenodo_api_url}/deposit/depositions/{entry_id}. `data` MUST be included as json.dump(data)
:param entry_id: str
deposition_id of the Zenodo entry
:param json_metadata: object
json object containing the metadata (compulsory fields) that are enclosed when a new entry is created.
:return: request.put method
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
headers = {"Content-Type": "application/json"}
# The metadata field is already created, just need to be updated.
# Thus the root 'metadata' key need to be kept, to indicate the field to be updated.
data = {"metadata": json_metadata}
return requests.put(url, data=json.dumps(data),
headers=headers, params=self.parameters)
def erase_entry(self, entry_id):
"""
Erase an entry/new version of an entry that HAS NOT BEEN published yet.
Any new upload/version will be first saved as 'draft' and not published until confirmation (i.e, requests.post)
DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}.
:param entry_id: str
deposition_id of the Zenodo entry to be erased
:return: request.delete method
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
return requests.delete(url, params=self.parameters)
def erase_file_entry(self, entry_id, file_id):
"""
Erase a file from an entry resource.
This method is intended to be used for substitution of files (deletion) within an entry by their correspondent
new versions.
DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}
:param entry_id: str
deposition_id of the Zenodo entry
:param file_id: str
Id of the files stored in Zenodo
:return: requests.delete method
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}"
return requests.delete(url, params=self.parameters)
def publish_entry(self, entry_id):
"""
Publishes an entry in (sandbox.)zenodo
POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.put method
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish"
return requests.post(url, params=self.parameters)
def new_version_entry(self, entry_id):
"""
Creates a new version of AN EXISTING entry resource.
POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion
:param entry_id: str
deposition_id of the Zenodo entry
:return: requests.post method
"""
url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion"
parameters = {'access_token': self.access_token}
return requests.post(url, params=parameters)
def fetch_community_entries(self, community_name='escape2020', results_per_query=100):
"""
Query the entries within a community.
GET method, previous modification of the query arguments, to {zenodo_api_url}/records
:param community_name: str
Community name. DEFAULT='escape2020'
:param results_per_query: int
Number of entries returned per call to the REST API. DEFAULT=100.
:return: requests.get method
"""
# https://developers.zenodo.org/#list36
update_query_args = {'communities': str(community_name),
'size': int(results_per_query)
}
self.parameters.update(update_query_args)
# Full answer
# content = requests.post(url, params=self.parameters)
# Answer items
# content.json().keys()
# Stats
# content.json()['aggregations']
# Total num of entries
# content.json()['hits']['total']
# Zenodo metadata of each entry
# [item['metadata'] for item in content.json()['hits']['hits']]
return requests.get(f"{self.zenodo_api_url}/records", params=self.parameters)
def fetch_community_entries_per_id(self, community_name='escape2020', results_per_query=100):
"""
Query the `entries ids` of all the entries within a community
:param community_name: str
Community name. DEFAULT='escape2020'
:param results_per_query: int
Number of entries returned per call to the REST API. DEFAULT=100.
:return: list
List containing the `id`s of each community entry
"""
return [entry['id'] for entry in
self.fetch_community_entries(community_name, results_per_query).json()['hits']['hits']]
def fetch_community_entries_per_title(self, community_name='escape2020', results_per_query=100):
"""
Query the title of all the entries within a community
:param community_name: str
Community name. DEFAULT='escape2020'
:param results_per_query: int
Number of entries returned per call to the REST API. DEFAULT=100.
:return: list
List containing the title of each community entry
"""
return [entry['metadata']['title'] for entry in
self.fetch_community_entries(community_name, results_per_query).json()['hits']['hits']]
def search_codemeta_file(self):
"""Check if a `codemeta.json` files exists in the ROOT directory of the project"""
# root_dir = find_root_directory()
root_dir = self.proj_root_dir
print(f'\nProject root directory {abspath(root_dir)}') # DEBUG
codemeta_file = root_dir / 'codemeta.json'
if codemeta_file.exists():
print("\n * Found codemeta.json file within the project !")
self.exist_codemeta_file = True
self.path_codemeta_file = codemeta_file
else:
print("\n ! codemeta.json file NOT found in the root directory of the project !")
def search_zenodo_json_file(self):
"""Check if a `.zenodo.json` files exists in the ROOT directory of the project"""
# root_dir = find_root_directory()
root_dir = self.proj_root_dir
zenodo_metadata_file = root_dir / '.zenodo.json'
if zenodo_metadata_file.exists():
print("\n * Found .zenodo.json file within the project !")
self.exist_zenodo_metadata_file = True
self.path_zenodo_metadata_file = zenodo_metadata_file
else:
print("\n ! .zenodo.json file NOT found in the root directory of the project !")
def conversion_codemeta2zenodo(self):
"""Perform the codemeta2zenodo conversion if a codemeta.json file is found"""
if self.exist_codemeta_file:
print("\n * Creating a .zenodo.json file from your codemeta.json file...")
self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
self.path_zenodo_metadata_file)
else:
pass
def test_upload_to_zenodo(self):
"""
`Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every
stage is the correct one.
Checks:
- The existence of a `.zenodo.json` file in the ROOT dir of the project
- If not, it checks if it exists a `codemeta.json` file
- If it exists it performs the codemeta2zenodo conversion
- If not, it exits the program
- The communication with Zenodo through its API to verify that:
- You can fetch an user entries
- You can create a new entry
- The provided zenodo metadata can be digested, and not errors appear
- Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED !
"""
# Search for the codemeta.json and the .zenodo.json files within the project
self.search_codemeta_file()
self.search_zenodo_json_file()
if not self.exist_zenodo_metadata_file:
if self.exist_codemeta_file:
self.conversion_codemeta2zenodo()
else:
print("\n ! NO codemeta.json NOR .zenodo.json file found. "
"Please add one to the ROOT directory of your project.")
sys.exit(-1)
print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
# 1 - Test connection
print("1 --> Testing communication with Zenodo...")
test_connection = self.fetch_user_entries()
if test_connection.status_code == 200:
print(" * Test connection status OK !")
else:
print(" ! ERROR while testing connection status\n", test_connection.json())
# 2 - Test new entry
print("2 --> Testing the creation of a dummy entry to (sandbox)Zenodo...")
new_entry = self.create_new_entry()
if new_entry.status_code == 201:
print(" * Test new entry status OK !")
else:
print(" ! ERROR while testing the creation of new entry\n", new_entry.json())
# 3 - Test upload metadata
print("3 --> Testing the ingestion of the Zenodo metadata...")
test_entry_id = new_entry.json()['id']
with open(self.path_zenodo_metadata_file) as file:
metadata_entry = json.load(file)
update_metadata = self.update_metadata_entry(test_entry_id,
json_metadata=metadata_entry)
if update_metadata.status_code == 200:
print(" * Update metadata status OK !")
pprint.pprint(metadata_entry)
else:
print(" ! ERROR while testing update of metadata\n",
update_metadata.json(), "\n", metadata_entry)
print(" ! Erasing dummy test entry...\n")
erase_error = self.erase_entry(test_entry_id)
if erase_error.status_code != 204:
print(f" !! ERROR erasing dummy test entry. Please erase it manually at\n "
f"{self.zenodo_api_url[:-4]}/deposit")
else:
print(" - Done.\n")
sys.exit(-1)
# 4 - Test delete entry
print("4 --> Testing the deletion of the dummy entry...")
delete_test_entry = self.erase_entry(test_entry_id)
if delete_test_entry.status_code == 204:
print(" * Delete test entry status OK !")
else:
print(" ! ERROR while deleting test entry\n", delete_test_entry.json())
print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
"You should not face any trouble when uploading a project to Zenodo - if you followed the "
"`OSSR how to publish tutorial`:\n"
"\t https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/contribute/publish_tutorial/#3-add-the-following-code-snippet \n"
"In case you do, please contact us !\n")