diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1080a7e41ef303945367989db39e82a9af0199f..da7f731b87b9d0caf2754aa1c2b849c34aeaa11e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - install - test - build_containers + - zenodo - deploy .junit_template: &junit_definition @@ -56,6 +57,22 @@ build_docker: only: - tags +deploy_zenodo: + stage: zenodo + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + before_script: + - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - eossr-upload-new-deposit --token $ZENODO_TOKEN --sandbox False --input-dir ./build +# - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + only: + - tags + pages: stage: deploy image: python:3.7-alpine @@ -67,4 +84,3 @@ pages: - public only: - master - diff --git a/examples/CI_code_snippets/1.ex_CI_build_docker_container.md b/examples/CI_code_snippets/1.ex_CI_build_docker_container.md new file mode 100644 index 0000000000000000000000000000000000000000..a7f3adc586720bb8a05068ad03868cf04bbb5b9b --- /dev/null +++ b/examples/CI_code_snippets/1.ex_CI_build_docker_container.md @@ -0,0 +1,31 @@ +# Build a Docker container during the CI process + + - Builds a Docker container during the CI process. + - Uploads the container to the GitLab container registry. + +**NOTE**. You should provide the Docker recipe (`Dockerfile`) and add it to a `Docker` directory in the root directory +of your project. + +```yaml +stages: + - build_container + + +build_docker: + stage: build_container + image: docker:19.03.12 + services: + - docker:19.03.12-dind + before_script: + - cat /etc/os-release # "Alpine Linux v3.12" + - apk add git + - export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'` + - echo $LAST_RELEASE + script: + - cd Docker # You should have added before your Dockerfile in this dir + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE . + - docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE + only: + - tags +``` \ No newline at end of file diff --git a/examples/CI_code_snippets/2.ex_CI_build_singularity_image.md b/examples/CI_code_snippets/2.ex_CI_build_singularity_image.md new file mode 100644 index 0000000000000000000000000000000000000000..1fbb7299b9ff63171f40434a3f101353cd10d85c --- /dev/null +++ b/examples/CI_code_snippets/2.ex_CI_build_singularity_image.md @@ -0,0 +1,40 @@ +# Build a Singularity image during the CI process + +This example is based on [this project](https://gitlab.com/singularityhub/gitlab-ci) (https://zenodo.org/record/3834833). +To be able to include the following code snippet into your `.gitlab-ci.yml` file, you should check the License +compatibility of your project and the origin one (an example can be found [here]( +https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/tree/master/.singularityci)). + + +This code snippet + + - Builds a Singularity image during the CI process. + - Makes available the created artifact (the image) to be used in the next CI stages. + +**PLEASE NOTE** + - You should provide the Singularity recipe (`Singularity`) and add it to a `Singularity` directory in the +root directory of your project. + - The image will be built using **Singularity v2.6**. + + +```yaml +stages: + - build_container + +build_singularity: + stage: build_container + image: singularityware/singularity:gitlab-2.6 + script: + - export IMAGE_NAME=$CI_PROJECT_NAME # Choose the name of the image + - singularity build $IMAGE_NAME.simg Singularity/Singularity + + - mkdir -p build && cp Singularity/*.simg build + - mkdir -p build && cp Singularity/Singularity* build + artifacts: + paths: + - build/Singularity.simg + - build/Singularity + only: + - tags + +``` \ No newline at end of file diff --git a/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md b/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md new file mode 100644 index 0000000000000000000000000000000000000000..76396e18bdab8496ed4592316289a62353cd436c --- /dev/null +++ b/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md @@ -0,0 +1,54 @@ +# Upload a new entry to the OSSR + + - Uses the GitLab CI to upload the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo community). + - Note that the CI will be only triggered with the creation of a new release. + - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as + a separate file. + - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check + that the released code will be successfully uploaded. + + +### Upload to Zenodo +```yaml +stages: + - deploy + +deploy_zenodo: + stage: deploy + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + before_script: + - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - eossr-upload-new-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + only: + - tags + +``` + + +### Upload to Sandbox Zenodo +```yaml +stages: + - deploy + +deploy_zenodo: + stage: deploy + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + before_script: + - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - eossr-upload-new-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID + only: + - tags + +``` diff --git a/examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md b/examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md new file mode 100644 index 0000000000000000000000000000000000000000..961cdc8261610d14bfd4adb5bc69a6df69abeaf1 --- /dev/null +++ b/examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md @@ -0,0 +1,69 @@ +# Upload a new version of an existing entry to the OSSR + + - Uses the GitLab CI to upload a **new version** of the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo + community). + - Note that the CI will be only triggered with the creation of a new release. + - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as + a separate file. + - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check + that the released code will be successfully uploaded. + +**NOTE**. +You should have saved the `deposit_id` of your project as a GitLab environment variable before the CI runs this stage. + +To do so: +1. Go to https://zenodo.org/deposit, +2. Click onto your just uploaded project, +3. From your browser search bar, **just** copy the number (your `deposit id`) that it is included in the https direction. + - ex: `https://zenodo.org/record/3884963` --> just copy `3884963`. +4. Save it as a new environment variable in your GitLab project. + * Go to your GitLab project. + * Click on `Settings` --> `CI/CD` --> `Variables` --> `Add variable` + * `KEY`=`$ZENODO_PROJECT_ID` and fill the value with the deposit id. + * For Sandbox Zenodo use `KEY` = `SANDBOX_ZENODO_PROJECT_ID` + + +### Upload to Zenodo +```yaml +stages: + - deploy + +deploy_zenodo: + stage: deploy + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + before_script: + - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + only: + - tags + +``` + + +### Upload to Sandbox Zenodo +```yaml +stages: + - deploy + +deploy_zenodo: + stage: deploy + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + before_script: + - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - eossr-upload-new-version-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $SANDBOX_ZENODO_PROJECT_ID + only: + - tags + +``` diff --git a/examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md b/examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md new file mode 100644 index 0000000000000000000000000000000000000000..e392a029d94f635a56fd620d0a0b2c3f7c456ad4 --- /dev/null +++ b/examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md @@ -0,0 +1,80 @@ +# Build a Singularity image, a Docker container and upload your project and images to the OSSR + +This code snippet will: + + - Builds a Docker container during the CI process. + - Builds a Singularity image during the CI process. + - Makes available the created artifacts (both images) to be used in the next CI stages. + - Uploads the next released version of the current project, together with both images, to the ESCAPE-OSSR + (The ESCAPE2020 Zenodo community). + +Have a look before to the examples in this same directory. + +```yaml +stages: + - build_container + - deploy + +build_singularity_image: + stage: build + image: singularityware/singularity:gitlab-2.6 + script: + # You should have added before your Singularity recipe in a Singularity dir + - singularity build $IMAGE_NAME.simg Singularity/Singularity + + - mkdir -p build && cp Singularity/*.simg build + - mkdir -p build && cp Singularity/Singularity* build + artifacts: + paths: + - build/Singularity.simg + - build/Singularity + only: + - tags + +build_docker_image: + stage: build + image: docker:19.03.12 + services: + - docker:19.03.12-dind + before_script: + - cat /etc/os-release # "Alpine Linux v3.12" + - apk add git + - export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'` + - echo $LAST_RELEASE + script: + - cd Docker + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE . + - docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE + + # Save docker image into a file to be uploaded in next stage + - docker save -o Docker_image_ESCAPE_template_project_$LAST_RELEASE.tar $CI_REGISTRY_IMAGE:$LAST_RELEASE + - cd .. && mkdir -p build && cp Docker/*.tar build + - ls build + artifacts: + paths: + - build/Docker_image_ESCAPE_template_project_*.tar + only: + - tags + + +deploy_zenodo: + stage: deploy + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1 + # The `dependencies` key (and field) is added to this stage ONLY because we have created + # in previous stages some artifacts + dependencies: + - build_singularity_image + - build_docker_image + before_script: + - test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR + script: + - mkdir -p build + - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL + - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - ls ./build + + - upload_new_version_deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + only: + - tags +```