Commit 10afa815 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Merge branch 'docker_ZenodoCI' into 'master'

Docker zenodoci

See merge request !10
parents 72692061 0e5fdba5
Pipeline #98700 failed with stages
in 56 seconds
stages:
- test
- deploy
- test_the_deploy
test_install:
stage: test
image: python:3.6.11-buster
script:
- apt-get -y update
- pip install .
only:
- branches
stage: test
image: python:3.6.11-buster
script:
- apt-get -y update
- pip install .
only:
- branches
deploy_zenodo:
stage: deploy
image: python:3.6.11-buster
#dependencies:
### Ideally to be used within a ci pipeline in where a container of the source code is build in a previous stage.
### You can have a look into https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/blob/master/.gitlab-ci.yml
# - build_image
before_script:
### 1 - Install dependencies in the image and upload the files to Zenodo
- apt-get -y update
stage: deploy
image: python:3.6.11-buster
#dependencies:
### Ideally to be used within a ci pipeline in where a container of the source code is build in a previous stage.
### You can have a look into https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/blob/master/.gitlab-ci.yml
# - build_image
before_script:
### 1 - Install dependencies in the image and upload the files to Zenodo
- apt-get -y update
### INFORMATION FOR THE USER; Python, pip and wget are already installed in the container
#- cat /etc/os-release # Debian GNU/Linux 10 (buster)
#- pip3 --version # pip 20.1.1
#- python3 --version # 3.6.11 as expected
- pip install requests numpy
- pip install . # Install ZenodoCI
### INFORMATION FOR THE USER; Python, pip and wget are already installed in the container
#- cat /etc/os-release # Debian GNU/Linux 10 (buster)
#- pip3 --version # pip 20.1.1
#- python3 --version # 3.6.11 as expected
- pip install requests numpy
- pip install . # Install ZenodoCI
### 2 - Test the communication to Zenodo - Uncomment if needed, or if already done it manually.
- test_connection_zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -r $CI_PROJECT_DIR
#- test_connection_zenodo -t $ZENODO_TOKEN -s False --repo_dir $CI_PROJECT_DIR
### 2 - Test the communication to Zenodo - Uncomment if needed, or if already done it manually.
- test_connection_zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR
#- test_connection_zenodo -t $ZENODO_TOKEN -s False --project_dir $CI_PROJECT_DIR
script:
### 3 - Get the last tag/release of the repository
- export REPOSITORY_NAME=zenodoci
- export REPOSITORY_BASE_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
script:
### 3 - Get the last tag/release of the repository
- export REPOSITORY_NAME=zenodoci
- export REPOSITORY_BASE_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
### 4 - Download the repository and move it to the build directory
### If no release is found/correctly parsed, the script will download the last commit pushed to the master branch
- mkdir -p build
- /bin/bash zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL
- ls ./build
### 4 - Download the repository and move it to the build directory
### If no release is found/correctly parsed, the script will download the last commit pushed to the master branch
- mkdir -p build
- /bin/bash zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_BASE_URL
- ls ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
- >
upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-directory ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO SANDBOX
- upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO
#- >
# upload_new_deposit
# --token $ZENODO_TOKEN
# --sandbox False
# --input-directory ./build
### 5 - To deploy a NEW DEPOSIT to ZENODO
#- >
# upload_new_deposit
# --token $ZENODO_TOKEN
# --sandbox False
# --input-dir ./build
### 5 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
upload_new_version_deposit
--token $SANDBOX_ZENODO_TOKEN
--sandbox True
--input-directory ./build
--deposit_id $DEPOSIT_ID_ZENODOCI
# --token $ZENODO_TOKEN
# --sandbox False
### 5 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
upload_new_version_deposit
--token $SANDBOX_ZENODO_TOKEN
--sandbox True
--input-dir ./build
--deposit_id $ZENODO_PROJECT_ID
# --token $ZENODO_TOKEN
# --sandbox False
only:
### Ideally this stage should be run only when a new release / tag of the source code is created, i.e., (- tags).
# The script is changed to check that the both `upload_new_deposit` and `upload_new_version_deposit` works nicely.
- tags
- branches # For testing
\ No newline at end of file
only:
### Ideally this stage should be run only when a new release / tag of the source code is created, i.e., (- tags).
# The script is changed to check that the both `upload_new_deposit` and `upload_new_version_deposit` works nicely.
- tags
- branches # For testing
test_deploy_from_container:
stage: test_the_deploy
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci
before_script:
- wget -q https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/raw/master/zenodoci/parse_last_release.sh
- pwd parse_last_release.sh # testing/debugging
- test_connection_zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR
script:
- pwd # testing/debugging
- ls # testing/debugging
- env # testing/debugging
- upload_new_deposit -h
- upload_new_version_deposit -h
- codemeta2zenodo -h
- mkdir -p build
- /bin/bash parse_last_release.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- ls ./build
- upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build
- upload_new_version_deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
only:
- tags
- branches # testing/debugging
FROM continuumio/miniconda3
ENV PATH=/opt/conda/bin/:${PATH}
SHELL [ "/bin/bash", "-c"]
RUN conda install git pip wget && \
# pip install git+https://gitlab.in2p3.fr/escape2020/wp3/zenodoci
git clone https://gitlab.in2p3.fr/escape2020/wp3/zenodoci.git
WORKDIR zenodoci
RUN pip install .
WORKDIR ..
......@@ -15,18 +15,105 @@ Library to manage an upload to Zenodo through its REST API.
$ pip install .
```
Then you would be able to run any of the following;
```sh
$ upload_new_deposit --token $ZENODO_TOKEN --sandbox False --input_dir <./path_proj_dir>
```
```sh
$ upload_new_version_deposit -t $ZENODO_TOKEN -s False -i <./your_proj_dir> -id <ZENODO_PROJ_ID>
```
```sh
$ test_connection_zenodo -t $ZENODO_TOKEN -s False --repo_dir <PROJ_DIR>
```
## _Quickstart_ - Test the full OSSR-CI and the communication to Zenodo
Please note that all the code developed in this project, together with the [codemeta2zenodo crosswalk](
https://gitlab.in2p3.fr/escape2020/wp3/codemeta2zenodo) are included and installed into a Docker container that can be
found at [the container registry](https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/container_registry) of this project.
1. Add a `codementa.json` file to your project.
**Fell free to `pull` it and use at any moment**.
## _Quickstart_ - Deploy your project to Zenodo (`one-click-build-and-publish`).
1. Include a `codementa.json` metadata file to your project.
- [CodeMeta generator](https://codemeta.github.io/codemeta-generator/).
2. Create a token at (sandbox)zenodo and add it as a variable to your project (see [below](#zenodo-token-gitlab-ci-environment-variable)).
3. **TEST** that you can communicate correctly with Zenodo. You will test and debug at the same time that all the
stages of the GitLabCI-Zenodo pipeline will work correctly.
```bash
$ python zenodoci/test_connection_zenodo.py --token YOUR_ZENODO_TOKEN --sandbox False
- The ZenodoCI pipeline will not work without one.
2. Create a token at [zenodo](https://zenodo.org/) and add it as an environment variable to your project (see
[below](#zenodo-token-gitlab-ci-environment-variable)).
3. Add in your `.gitlab-ci.yml` file the following stage to deploy your project into Zenodo.
4. Create a release.
The code to be added to the `.gitlab-ci.yml` file is the following (**COMMON FOR EVERY PROJECT**).
### To create a new entry or deposit
```
stages:
- `(...) all your CI stages (...)`
- deploy_to_zenodo
`(...) All the code to be run in your CI pipeline (...)`
deploy:
stage: deploy_to_zenodo
image: gitlab-registry.in2p3.fr/escape2020/wp3/zenodoci
before_script:
- wget -q https://gitlab.in2p3.fr/escape2020/wp3/zenodoci/-/raw/master/zenodoci/parse_last_release.sh
- test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
script:
- mkdir -p build
- /bin/bash parse_last_release.sh $CI_PROJECT_NAME $CI_PROJECT_URL
- ls ./build
- upload_new_deposit --token $SANDBOX_ZENODO_TOKEN --sandbox True --input-dir ./build
only:
- tags
```
### To create a new version of an existing entry
Substitute
```
- upload_new_version_deposit -t $ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
```
for
```
- upload_new_version_deposit -t $ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
```
and please check the correct indentation on the file.
## Zenodo token & GitLab CI environment variable
To allow GitLab communicate with Zenodo through their APIs, a personal access token must be created and included into
the GitLab project (as a **masked !** environment variable).
Please note that (to date) the token can be assigned **only to a single** Zenodo account. To create the token:
- Go to [zenodo.org](https://zenodo.org/).
- Click on `Account` --> `Applications` --> `Personal access token` --> `New token` and Select the desired `Scopes`.
This token will be passed later in the deployment stage of the ZenodoCI pipeline (the same that you should also include
into your project to automatise the upload).
For not sharing publicly your personal token, you should create an environment variable in your GitLab repository.
This way, the token could be used as a variable without revealing its value. To create an an environment variable:
- Go to your GitLab project.
- Click on `Settings` --> `CI/CD` --> `Variables` --> `Add variable` --> Fill the fields --> **Mask your variable(s) !!**
Please name your environment variables as follows (so that no changes should be done to the `.gitlab-ci.yml` file);
- `ZENODO_TOKEN` or `SANDBOX_ZENODO_TOKEN`
- `ZENODO_PROJECT_ID`
so that environment variable(s) should look like:
```sh
$ upload_new_deposit --token $ZENODO_TOKEN --sandbox False --input-directory <./build>
$ upload_new_version_deposit -t $ZENODO_TOKEN -s False -i <./your_proj_dir> -id $ZENODO_PROJECT_ID
```
# Contact
[Contact us](mailto:garcia@lapp.in2p3.fr) for any question or doubt related with the current library, the ESCAPE project,
the metadata schema context or the integration of any project to Zenodo, Gitlab or the ESCAPE OSSR.
<!-- COMMENT
## Continuous Integration to Zenodo
The library was developed specifically to perform a deploy stage (to the Zenodo repository) within a GitLab CI
......@@ -69,29 +156,4 @@ how to easily create a `codemeta.json` file.
Last but not least ! Please note that during the CI pipeline the `zenodoci` module will search for a `codemeta.json` file
and will automatically create the equivalent file to provide metadata to the Zenodo repository. The `.zenodo.json` file
will contain the exactly same information that in the `codemeta.json` file but using the Zenodo syntax.
## Zenodo token & GitLab CI environment variable
To connect the GitLab repository with Zenodo in an autonomous way, a personal access token must be created. This token
is assigned to a **single** Zenodo account, and it will allow the interaction with
(sandbox.)zenodo through its API. To create the token:
- Go to (sandbox)zenodo.org
- Account --> Applications --> Personal access token --> New token.
This token will be passed later in the deployment stage of the CI pipeline. For not sharing publicly
your personal token, you should create an environment variable in your GitLab repository. This way, the token could be
used as a variable without revealing its value. To create an an environment variable:
- Go to your GitLab repository.
- Settings --> CI/CD --> Variables --> Add variable --> Fill the fields --> Mask your variable(s) !!
The environment variable will look like this:
```sh
$ python zenodoci/upload_new_deposit.py --input-directory build --token $ZENODO_TOKEN --sandbox False
```
# Contact
[Contact us](mailto:garcia@lapp.in2p3.fr) for any question or doubt related with the current library, the ESCAPE project,
the metadata schema context or the integration of any project to Zenodo, Gitlab or the ESCAPE OSSR.
\ No newline at end of file
-->
\ No newline at end of file
......@@ -39,12 +39,6 @@
}
],
"softwareRequirements": [
{
"@type": "SoftwareApplication",
"identifier": "numpy",
"name": "numpy",
"softwareVersion": "1.18.1"
},
{
"@type": "SoftwareApplication",
"identifier": "requests",
......@@ -56,12 +50,6 @@
"identifier": "pyjson",
"name": "json",
"softwareVersion": ">=1.3.0"
},
{
"@type": "SoftwareApplication",
"identifier": "pyyaml",
"name": "pyyaml",
"softwareVersion": ">=5.3.1"
}
],
"maintainer": {
......
......@@ -24,8 +24,8 @@ def main():
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--repo_dir', '-r', action='store',
dest='repository_dir',
parser.add_argument('--project_dir', '-p', action='store',
dest='project_dir',
help='Path to the root directory of the directory to be uploaded.'
'DEFAULT; assumed to be on it, i.e., "./"',
default='./'
......@@ -35,7 +35,7 @@ def main():
zenodo = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag,
repo_root_dir=args.repository_dir
proj_root_dir=args.project_dir
)
zenodo.test_upload_to_zenodo()
......
......@@ -60,7 +60,7 @@ def main():
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
parser.add_argument('--input-dir', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
......
......@@ -59,7 +59,7 @@ def main():
'If True connects with Zenodo. If False with Sandbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
parser.add_argument('--input-dir', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
......
......@@ -11,7 +11,7 @@ from codemeta2zenodo_crosswalk.codemeta2zenodo import parse_codemeta_and_write_z
class ZenodoAPI:
def __init__(self, access_token, sandbox=True, repo_root_dir='.'):
def __init__(self, access_token, sandbox=True, proj_root_dir='.'):
"""
Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
......@@ -41,11 +41,11 @@ class ZenodoAPI:
self.zenodo_api_url = zenodo_api_url
self.access_token = access_token
self.repo_root_dir = Path(repo_root_dir)
self.proj_root_dir = Path(proj_root_dir)
self.exist_codemeta_file = False
self.path_codemeta_file = repo_root_dir
self.path_codemeta_file = proj_root_dir
self.exist_zenodo_metadata_file = False
self.path_zenodo_metadata_file = repo_root_dir
self.path_zenodo_metadata_file = proj_root_dir
def fetch_user_entries(self):
"""
......@@ -203,7 +203,7 @@ class ZenodoAPI:
"""Check if a `codemeta.json` files exists in the ROOT directory of the project"""
# root_dir = find_root_directory()
root_dir = self.repo_root_dir
root_dir = self.proj_root_dir
print(f'Repo root directory {root_dir}') # DEBUG
codemeta_file = root_dir / 'codemeta.json'
......@@ -219,7 +219,7 @@ class ZenodoAPI:
"""Check if a `.zenodo.json` files exists in the ROOT directory of the project"""
# root_dir = find_root_directory()
root_dir = self.repo_root_dir
root_dir = self.proj_root_dir
zenodo_metadata_file = root_dir / '.zenodo.json'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment