Commit 89149a56 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

clean repo and update CI

parent 765598dc
stages: stages:
- install
- test - test
- testContainer
- build - build
- deploy - deploy
build_test_project: install_project:
stage: test stage: install
# Please notice that we are using a different image just for easier syntax in the testing of the building and # Please notice that we are using a different image just for easier syntax in the testing of the building and
# testing of the repository # testing of the repository
image: continuumio/miniconda3:latest image: continuumio/miniconda3:latest
script: script:
- . /opt/conda/etc/profile.d/conda.sh - . /opt/conda/etc/profile.d/conda.sh
- conda env create -f environment.yml - conda env create -f environment.yml
- conda activate escape_template - conda activate escape_template
- python setup.py install - pip install .
- pytest . - square_number -h
- square_number -i -12.5
only:
- branches
test_project:
stage: test
image: python:3.6-buster
script:
- pip install pytest
- pip install .
- pytest .
only:
- branches
test_docker_registry: test_docker_registry:
stage: testContainer # Test of last container in registry
image: gitlab-registry.in2p3.fr/escape2020/wp3/template_project_escape/escape_template_project:v1.0 stage: test
script: image: gitlab-registry.in2p3.fr/escape2020/wp3/template_project_escape/escape_template_project:v1.0
- source activate escape_template script:
- square_number -i -12.5 - source activate escape_template
only: - square_number -i -12.5
- tags except:
- master
- tags
build_image: build_singularity_image:
stage: build # Example of Singularity image (of the project) built during the CI
image: singularityware/singularity:gitlab-2.6 stage: build
script: image: singularityware/singularity:gitlab-2.6
# Build the container! script:
- /bin/bash .gitlabci/build.sh Singularity # Build the container!
- /bin/bash .singularityci/build.sh Singularity/Singularity
- mkdir -p build && cp *.simg build - mkdir -p build && cp *.simg build
- mkdir -p build && cp Singularity* build - mkdir -p build && cp Singularity* build
artifacts:
paths:
- build/Singularity.simg
- build/Singularity
only:
- tags
artifacts: build_docker_image:
paths: # Example of Docker container (of the project) built - and uploaded to a registry - during the CI
- build/Singularity.simg stage: build
- build/Singularity image: docker:19.03.12
only: services:
# - master - docker:19.03.12-dind
- tags before_script:
- cat /etc/os-release # "Alpine Linux v3.12"
- apk add git
- export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'`
- echo $LAST_RELEASE
script:
- cd Docker
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE .
- docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE
only:
- tags
deploy_zenodo: deploy_zenodo:
stage: deploy stage: deploy
image: python:3.6.11-buster image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci:v1.2
dependencies: # The dependencies key is added to this stage ONLY because we have created in a previous stage some other artifacts
- build_image dependencies:
script: - build_singularity_image
# 1 - Install dependencies in the image and upload the files to Zenodo before_script:
- apt-get -y update - test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
script:
### python, pip and wget are already installed in the container - mkdir -p build
#- cat /etc/os-release # Debian GNU/Linux 10 (buster) - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
#- pip3 --version # pip 20.1.1 - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
#- python3 --version # 3.6.11 as the tag of the image specifies - ls ./build
- pip3 install requests
### 2 - Get the last tag/release of the repository
- export REPOSITORY_NAME=template_project_escape
- export REPOSITORY_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
### 3 - Download the repository and move it to the build directory
- mkdir -p build
- /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_URL
- ls ./build
### 4 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $SANDBOX_ZENODO_TOKEN
# --sandbox True
# --input-directory ./build
### 4 - To deploy a NEW DEPOSIT to ZENODO
#- >
# python3 .zenodoci/upload_new_deposit.py
# --token $ZENODO_TOKEN
# --sandbox False
# --input-directory ./build
### 4 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
--sandbox False
--token $ZENODO_TOKEN
--deposit_id $DEPOSIT_ID_ESCAPE_TEMPLATE
only: - upload_new_version_deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
- tags only:
- tags
# README _.gitlabci_ library # README _.singularityci_ module
[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3834833.svg)](https://doi.org/10.5281/zenodo.3834833) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3834833.svg)](https://doi.org/10.5281/zenodo.3834833)
...@@ -7,11 +7,11 @@ ...@@ -7,11 +7,11 @@
## Building Singularity Containers using the continuous integration in GitLab ## Building Singularity Containers using the continuous integration in GitLab
The source code contained in this folder is based on the following The source code contained in this folder is based on the following
[GitLab-CI repository](https://gitlab.com/singularityhub/gitlab-ci). You can find a very didactic **tutorial** of how [GitLab-CI project](https://gitlab.com/singularityhub/gitlab-ci). You can find a very didactic **tutorial** of how
to implement this code [here](https://vsoch.github.io/2018/gitlab-singularity-ci/). to implement this code [here](https://vsoch.github.io/2018/gitlab-singularity-ci/).
This software (composed of the files within this directory **AND** part of the the `.gitlab-ci.yml` file in the This software (composed of the files within this directory **AND** part of the the `.gitlab-ci.yml` file in the
root directory - the `build_image` stage) provide the necessary tools to: root directory - the `build_singularity_image` stage) provide the necessary tools to:
- Use a container image stored in the DockerHub site to create a Singularity container of your repository - Use a container image stored in the DockerHub site to create a Singularity container of your repository
(thus no need of having installed the Singularity source code), and upload it directly to your GitLab repository. (thus no need of having installed the Singularity source code), and upload it directly to your GitLab repository.
- The Singularity recipe must be provided, of course. - The Singularity recipe must be provided, of course.
......
{
"metadata": {
"access_right": "open",
"communities": [
{
"identifier": "escape2020"
}
],
"creators": [
{
"affiliation": "LAPP, CNRS",
"name": "Vuillaume, Thomas",
"orcid": "0000-0002-5686-2078"
},
{
"affiliation": "LAPP, CNRS",
"name": "Garcia, Enrique",
"orcid": "0000-0003-2224-4594"
}
],
"description": "A template repository for the ESCAPE project",
"grants": [
{
"id": "10.13039/501100000780::824064"
}
],
"keywords": [
"template"
],
"language": "eng",
"license": "GPL-3.0+",
"notes": "Release Notes: Documentation and implementation of the last version for CI/CD to Zenodo",
"publication_date": "2019-12-12",
"title": "ESCAPE template project",
"upload_type": "software",
"version": "v2.0"
}
}
\ No newline at end of file
MIT License
Copyright (c) 2020, ZenodoCI developers.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# README _.zenodoci_
**PLEASE HAVE A LOOK TO THE LICENSING SECTION BELOW BEFORE IMPLEMENTING ANY PART OF THIS CODE INTO YOURS !!**
Library to manage an upload to Zenodo through its REST API.
## _Quickstart_ - Test the full OSSR-CI and the communication to Zenodo
1. Add a `codementa.json` file to your project.
- [CodeMeta generator](https://codemeta.github.io/codemeta-generator/).
2. Create a token at (sandbox)zenodo and add it as a variable to your project (see [below](#zenodo-token-gitlab-ci-environment-variable)).
3. **TEST** that you can communicate correctly with Zenodo. You will test and debug at the same time that all the
stages of the GitLabCI-Zenodo pipeline will work correctly.
```bash
$ python .zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
```
## Continuous Integration to Zenodo
The source code contained in this folder is based on the [ZenodoCI](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci)
project. The library was developed specifically to perform a deploy stage (to the Zenodo repository) in a GitLab CI
pipeline that **could be implemented in any external project**.
The library (composed of the scripts within the `.zenodoci` directory) provides a module to handle the upload of
specified file(s) to the [ESCAPE2020 Zenodo community](https://zenodo.org/communities/escape2020/). Please **JUST**
upload stable versions/releases of source code and/or image
containers!
A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the
GitLabCI pipeline.
The `deploy` stage in the CI pipeline (see the `.gitlab-ci.yml` file) will make use of the `zenodoapi` library and
the built Singularity container created in the previous CI stage (check the `.gitlabci` directory too) to:
- Either upload the desired file(s) to the ESCAPE community in Zenodo.
- Either upload a new version of an existing entry to Zenodo.
A `codemeta.json` metadata file (see below) **MUST BE ADDED** before uploading an entry to Zenodo or triggering the GitLabCI pipeline.
Also, depending on the case, the corresponding python script (`upload_new_deposit.py` or `upload_new_version_of_deposit.py`)
must be adapted and included into the `.gitlab-ci.yml` file with its corresponding arguments (examples are shown in the yml file).
#### **Use of new metadata context ! Please check the news !**
We are no longer supporting the use of a `repository_information.json` file to provide metadata to Zenodo.
We are currently moving to a [CodeMeta metadata context](https://codemeta.github.io/).
This metadata standard provides a complete metadata schema context supported by many other services and search engines.
Adding a single `codemeta.json` file to the root directory of your project will be enough ! Please check out the
[ESCAPE metadata template](https://gitlab.in2p3.fr/escape2020/wp3/escape_metadata_template) project for a _quickstart_ on
how to easily create a `codemeta.json` file.
Last but not least ! Please note that during the CI pipeline the `.zenodoci` module will search for a `codemeta.json` file
and will automatically create the equivalent file to provide metadata to the Zenodo repository. The `.zenodo.json` file
will contain the exactly same information that in the `codemeta.json` file but using the Zenodo syntax.
## Zenodo token & GitLab CI environment variable
To connect the GitLab repository with Zenodo in an autonomous way, a personal access token must be created. This token
is assigned to a **single** Zenodo account, and it will allow the interaction with
(sandbox.)zenodo through its API. To create the token:
- Go to (sandbox)zenodo.org
- Account --> Applications --> Personal access token --> New token.
This token will be passed later in the deployment stage of the CI pipeline. For not sharing publicly
your personal token, you should create an environment variable in your GitLab repository. This way, the token could be
used as a variable without revealing its value. To create an an environment variable:
- Go to your GitLab repository.
- Settings --> CI/CD --> Variables --> Add variable --> Fill the fields --> Mask your variable(s) !!
The environment variable will look like this:
```sh
$ python .zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
```
-------------
# License of the `template_project_repository`:
The `template_project_repository` contains code from different projects. This 'mixing' can be done because:
- Both 'parent' projects are Open Source.
- None of the original licenses are copy-left.
- Both `BSD 3-Clause` and `MIT`, are permissive licenses. All of these points mean that source code distributed with
a BSD 3-Clause license can be included in a project with a MIT license - and of course be re-distributed under `MIT`:
- Note however, that the **original licenses** must be included in the resulting project and mentioned in the
LICENSE/documentation files.
**PLEASE TAKE THE TIME TO CHECK AND VERIFY LICENSES AND THEIR COMPATIBILITIES**
\ No newline at end of file
#!/usr/bin/env bash
REPOSITORY_NAME="$1"
REPOSITORY_BASE_URL="$2"
LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'`
if [ -z "$LAST_RELEASE" ]; then
echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"
wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip
mv $REPOSITORY_NAME-master.zip ./build
else
echo "$LAST_RELEASE tag / release found !"
wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip
mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
fi
#!/usr/bin/env python
# E. Garcia Nov 2020
# Module to test the connection and the upload of new entries/version to Zenodo.
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
if __name__ == '__main__':
# Required arguments
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
args = parser.parse_args()
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag
)
zenodo.test_upload_to_zenodo()
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from utils_zenodoci import (parse_codemeta_and_write_zenodo_metadata_file,
find_root_directory
)
def create_zenodo_metadata(metadata_filename):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
param metadata_filename: str
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
root_dir = find_root_directory()
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(files_json)
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
args = parser.parse_args()
zenodo = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = zenodo.create_new_entry()
if new_entry.status_code < 399:
deposition_id = new_entry.json()['id']
doi = new_entry.json()['metadata']['prereserve_doi']['doi']
print(f"Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
else:
print(new_entry.json())
# 2 - upload files
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = zenodo.upload_file_entry(
deposition_id,
name_file=file,
path_file=full_path_file
)
print(f"File {file} correctly uploaded !\n", new_upload)
# 3 - Create the zenodo metadata file from a codemeta.json file
zenodo_metadata_filename = '.zenodo.json'
create_zenodo_metadata(zenodo_metadata_filename)
# And upload the repository metadata
with open(zenodo_metadata_filename) as json_file:
entry_metadata = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = zenodo.update_metadata_entry(
deposition_id,
data=entry_metadata
)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !")
else:
print(f"Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
update_entry.json())
# 4 - publish entry - to publish the entry, uncomment the two lone below
# publish = zenodo.publish_entry(deposition_id)
# print(publish.json())
print("New deposit correctly published !")
print(f"The new doi should look like 10.5281/{deposition_id}. However please")
print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
#!/usr/bin/env python
# Enrique Garcia. Aug '20
# email garcia 'at' lapp.in2p3.fr
import os
import json
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from utils_zenodoci import (parse_codemeta_and_write_zenodo_metadata_file,
find_root_directory
)
def create_zenodo_metadata(metadata_filename):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
param metadata_filename: str
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
root_dir = find_root_directory()
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(files_json)
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
parser.add_argument('--deposit_id', '-id', type=str,
dest='deposit_id',
help='deposit_id of the deposit that is going to be updated by a new version',
required=True)
args = parser.parse_args()
zenodo = ZenodoAPI(