From 02787c4d90f6d513d47499df642299d842355fa6 Mon Sep 17 00:00:00 2001
From: vuillaut <thomas.vuillaume@gmail.com>
Date: Wed, 15 Sep 2021 01:20:05 +0200
Subject: [PATCH] documentation for notebooks and snippets in markdown

---
 .gitlab-ci.yml                                |  3 +-
 docs/conf.py                                  | 22 +++--
 docs/doc_requirements.txt                     |  4 +
 docs/index.rst                                |  4 +-
 docs/snippets.rst                             |  8 ++
 .../1.ex_CI_build_docker_container.md         | 31 +++++++
 .../2.ex_CI_build_singularity_image.md        | 40 ++++++++++
 docs/snippets/3.ex_CI_upload_new_deposit.md   | 54 +++++++++++++
 .../4.ex_CI_upload_new_version_deposit.md     | 69 ++++++++++++++++
 .../5.ex_CI_build_image_and_upload_OSSR.md    | 80 +++++++++++++++++++
 examples/notebooks/Harvest_zenodo.ipynb       | 22 +++--
 examples/notebooks/ossr_api.ipynb             | 10 ++-
 12 files changed, 329 insertions(+), 18 deletions(-)
 create mode 100644 docs/doc_requirements.txt
 create mode 100644 docs/snippets.rst
 create mode 100644 docs/snippets/1.ex_CI_build_docker_container.md
 create mode 100644 docs/snippets/2.ex_CI_build_singularity_image.md
 create mode 100644 docs/snippets/3.ex_CI_upload_new_deposit.md
 create mode 100644 docs/snippets/4.ex_CI_upload_new_version_deposit.md
 create mode 100644 docs/snippets/5.ex_CI_build_image_and_upload_OSSR.md

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2c0b01bb..aeb8dd05 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -77,8 +77,7 @@ pages:
     stage: deploy
     image: python:3.7-alpine
     script:
-      - pip install -U sphinx
-      - pip install sphinx_rtd_theme
+      - pip install -r docs/doc_requirements.txt
       - sphinx-build -b html docs public
     artifacts:
       paths:
diff --git a/docs/conf.py b/docs/conf.py
index ad557076..ac3ef95f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -13,14 +13,21 @@
 import os
 import sys
 import shutil
+import glob
 
 
 sys.path.insert(0, os.path.abspath('..'))
+
+# Notebooks
 notebook_dir = '../examples/notebooks/'
-if not os.path.exists('examples'):
-    os.mkdir('examples')
+os.makedirs('examples', exist_ok=True)
 [shutil.copy(notebook_dir + file, 'examples') for file in os.listdir(notebook_dir) if file.endswith('.ipynb')]
 
+# Snippets
+snippets_doc_dir = 'snippets'
+os.makedirs(snippets_doc_dir, exist_ok=True)
+for filename in glob.glob('../examples/CI_code_snippets/*.md'):
+    shutil.copy(filename, snippets_doc_dir)
 
 
 # -- Project information -----------------------------------------------------
@@ -40,20 +47,23 @@ extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.napoleon',
     'nbsphinx',
+    'myst_parser',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
 # Parsers
-source_parsers = {
-   '.md': 'recommonmark.parser.CommonMarkParser',
-}
+# source_parsers = {
+# }
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
-source_suffix = ['.rst', '.md']
+source_suffix = [
+    '.rst',
+    '.md',
+]
 
 
 # List of patterns, relative to source directory, that match files and
diff --git a/docs/doc_requirements.txt b/docs/doc_requirements.txt
new file mode 100644
index 00000000..b53c7566
--- /dev/null
+++ b/docs/doc_requirements.txt
@@ -0,0 +1,4 @@
+- sphinx
+- nbsphinx
+- sphinx_rtd_theme
+- myst-parser
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
index 8401bb3a..5521c98d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -7,7 +7,7 @@ Welcome to eossr's documentation!
 =================================
 
 .. toctree::
-   :maxdepth: 4
+   :maxdepth: 2
    :glob:
    :caption: Contents:
 
@@ -15,7 +15,7 @@ Welcome to eossr's documentation!
    license
    docstring
    examples
-
+   snippets
 
 
 Indices and tables
diff --git a/docs/snippets.rst b/docs/snippets.rst
new file mode 100644
index 00000000..b5709dd1
--- /dev/null
+++ b/docs/snippets.rst
@@ -0,0 +1,8 @@
+Snippets
+========
+
+.. toctree::
+   :maxdepth: 2
+   :glob:
+
+   snippets/*
\ No newline at end of file
diff --git a/docs/snippets/1.ex_CI_build_docker_container.md b/docs/snippets/1.ex_CI_build_docker_container.md
new file mode 100644
index 00000000..a7f3adc5
--- /dev/null
+++ b/docs/snippets/1.ex_CI_build_docker_container.md
@@ -0,0 +1,31 @@
+# Build a Docker container during the CI process
+
+ - Builds a Docker container during the CI process.
+ - Uploads the container to the GitLab container registry. 
+ 
+**NOTE**. You should provide the Docker recipe (`Dockerfile`) and add it to a `Docker` directory in the root directory 
+of your project.  
+
+```yaml
+stages:
+  - build_container
+
+
+build_docker:
+  stage: build_container
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  before_script:
+    - cat /etc/os-release  # "Alpine Linux v3.12"
+    - apk add git
+    - export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'`
+    - echo $LAST_RELEASE
+  script:
+    - cd Docker  # You should have added before your Dockerfile in this dir
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE .
+    - docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE
+  only:
+    - tags
+```   
\ No newline at end of file
diff --git a/docs/snippets/2.ex_CI_build_singularity_image.md b/docs/snippets/2.ex_CI_build_singularity_image.md
new file mode 100644
index 00000000..1fbb7299
--- /dev/null
+++ b/docs/snippets/2.ex_CI_build_singularity_image.md
@@ -0,0 +1,40 @@
+# Build a Singularity image during the CI process
+
+This example is based on [this project](https://gitlab.com/singularityhub/gitlab-ci) (https://zenodo.org/record/3834833).
+To be able to include the following code snippet into your `.gitlab-ci.yml` file, you should check the License 
+compatibility of your project and the origin one (an example can be found [here](
+https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape/-/tree/master/.singularityci)).
+
+
+This code snippet 
+
+ - Builds a Singularity image during the CI process.
+ - Makes available the created artifact (the image) to be used in the next CI stages. 
+ 
+**PLEASE NOTE** 
+ - You should provide the Singularity recipe (`Singularity`) and add it to a `Singularity` directory in the 
+root directory of your project.
+ - The image will be built using  **Singularity v2.6**.
+
+
+```yaml
+stages:
+  - build_container
+
+build_singularity:
+  stage: build_container
+  image: singularityware/singularity:gitlab-2.6
+  script:
+    - export IMAGE_NAME=$CI_PROJECT_NAME  # Choose the name of the image
+    - singularity build $IMAGE_NAME.simg Singularity/Singularity
+
+    - mkdir -p build && cp Singularity/*.simg build
+    - mkdir -p build && cp Singularity/Singularity* build
+  artifacts:
+    paths:
+      - build/Singularity.simg
+      - build/Singularity
+  only:
+    - tags
+
+```   
\ No newline at end of file
diff --git a/docs/snippets/3.ex_CI_upload_new_deposit.md b/docs/snippets/3.ex_CI_upload_new_deposit.md
new file mode 100644
index 00000000..76396e18
--- /dev/null
+++ b/docs/snippets/3.ex_CI_upload_new_deposit.md
@@ -0,0 +1,54 @@
+# Upload a new entry to the OSSR 
+
+ - Uses the GitLab CI to upload the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo community).
+    - Note that the CI will be only triggered with the creation of a new release.
+ - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as 
+ a separate file.
+ - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check
+ that the released code will be successfully uploaded. 
+
+
+### Upload to Zenodo
+```yaml
+stages:
+ - deploy 
+
+deploy_zenodo:
+  stage: deploy
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  before_script:
+    - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - eossr-upload-new-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
+  only:
+    - tags
+
+``` 
+
+
+### Upload to Sandbox Zenodo
+```yaml
+stages:
+ - deploy 
+
+deploy_zenodo:
+  stage: deploy
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  before_script:
+    - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - eossr-upload-new-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID
+  only:
+    - tags
+
+``` 
diff --git a/docs/snippets/4.ex_CI_upload_new_version_deposit.md b/docs/snippets/4.ex_CI_upload_new_version_deposit.md
new file mode 100644
index 00000000..961cdc82
--- /dev/null
+++ b/docs/snippets/4.ex_CI_upload_new_version_deposit.md
@@ -0,0 +1,69 @@
+# Upload a new version of an existing entry to the OSSR 
+
+ - Uses the GitLab CI to upload a **new version** of the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo 
+ community).
+     - Note that the CI will be only triggered with the creation of a new release.
+ - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as 
+ a separate file.
+ - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check
+ that the released code will be successfully uploaded. 
+ 
+**NOTE**.
+You should have saved the `deposit_id` of your project as a GitLab environment variable before the CI runs this stage.
+
+To do so:
+1. Go to https://zenodo.org/deposit,
+2. Click onto your just uploaded project,
+3. From your browser search bar, **just** copy the number (your `deposit id`) that it is included in the https direction.
+   - ex: `https://zenodo.org/record/3884963` --> just copy `3884963`.
+4. Save it as a new environment variable in your GitLab project. 
+    * Go to your GitLab project. 
+    * Click on `Settings` --> `CI/CD` --> `Variables` --> `Add variable` 
+    * `KEY`=`$ZENODO_PROJECT_ID` and fill the value with the deposit id.
+    * For Sandbox Zenodo use `KEY` = `SANDBOX_ZENODO_PROJECT_ID`
+
+
+### Upload to Zenodo
+```yaml
+stages:
+ - deploy 
+
+deploy_zenodo:
+  stage: deploy
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  before_script:
+    - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
+  only:
+    - tags
+
+``` 
+
+
+### Upload to Sandbox Zenodo
+```yaml
+stages:
+ - deploy 
+
+deploy_zenodo:
+  stage: deploy
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  before_script:
+    - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - eossr-upload-new-version-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $SANDBOX_ZENODO_PROJECT_ID
+  only:
+    - tags
+
+``` 
diff --git a/docs/snippets/5.ex_CI_build_image_and_upload_OSSR.md b/docs/snippets/5.ex_CI_build_image_and_upload_OSSR.md
new file mode 100644
index 00000000..e392a029
--- /dev/null
+++ b/docs/snippets/5.ex_CI_build_image_and_upload_OSSR.md
@@ -0,0 +1,80 @@
+# Build a Singularity image, a Docker container and upload your project and images to the OSSR
+
+This code snippet will:
+
+ - Builds a Docker container during the CI process.
+ - Builds a Singularity image during the CI process.
+ - Makes available the created artifacts (both images) to be used in the next CI stages.
+ - Uploads the next released version of the current project, together with both images, to the ESCAPE-OSSR 
+ (The ESCAPE2020 Zenodo community).
+
+Have a look before to the examples in this same directory.
+
+```yaml
+stages:
+ - build_container
+ - deploy
+
+build_singularity_image:
+  stage: build
+  image: singularityware/singularity:gitlab-2.6
+  script:
+    # You should have added before your Singularity recipe in a Singularity dir
+    - singularity build $IMAGE_NAME.simg Singularity/Singularity
+
+    - mkdir -p build && cp Singularity/*.simg build
+    - mkdir -p build && cp Singularity/Singularity* build
+  artifacts:
+    paths:
+      - build/Singularity.simg
+      - build/Singularity
+  only:
+    - tags
+
+build_docker_image:
+  stage: build
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  before_script:
+    - cat /etc/os-release  # "Alpine Linux v3.12"
+    - apk add git
+    - export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $CI_PROJECT_URL.git | tail -n1 | sed 's/.*\///'`
+    - echo $LAST_RELEASE
+  script:
+    - cd Docker
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t $CI_REGISTRY_IMAGE:$LAST_RELEASE .
+    - docker push $CI_REGISTRY_IMAGE:$LAST_RELEASE
+
+    # Save docker image into a file to be uploaded in next stage
+    - docker save -o Docker_image_ESCAPE_template_project_$LAST_RELEASE.tar $CI_REGISTRY_IMAGE:$LAST_RELEASE
+    - cd .. && mkdir -p build && cp Docker/*.tar build
+    - ls build
+  artifacts:
+    paths:
+      - build/Docker_image_ESCAPE_template_project_*.tar
+  only:
+    - tags
+
+
+deploy_zenodo:
+  stage: deploy
+  image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.1
+  # The `dependencies` key (and field) is added to this stage ONLY because we have created 
+  #  in previous stages some artifacts
+  dependencies:
+    - build_singularity_image
+    - build_docker_image
+  before_script:
+    - test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR
+  script:
+    - mkdir -p build
+    - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL
+    - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi
+    - ls ./build
+
+    - upload_new_version_deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID
+  only:
+    - tags
+```
diff --git a/examples/notebooks/Harvest_zenodo.ipynb b/examples/notebooks/Harvest_zenodo.ipynb
index 40b8387b..d7e137c4 100644
--- a/examples/notebooks/Harvest_zenodo.ipynb
+++ b/examples/notebooks/Harvest_zenodo.ipynb
@@ -1,12 +1,18 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "83cfb991",
+   "metadata": {},
+   "source": [
+    "# Example of protocols to harvest metadata from Zenodo"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "b0fcf1bf",
    "metadata": {},
    "source": [
-    "<h1><center> <font size=\"36\"> Example of protocols to harvest metadata from Zenodo </font> </center></h1>\n",
-    "\n",
     "---------------------\n",
     "#### Notebook outline \n",
     " - Zenodo OAI-PMH protocol\n",
@@ -23,7 +29,7 @@
    "id": "2529eacc",
    "metadata": {},
    "source": [
-    "## TL;DR: Pros and cons of each method\n",
+    "## Pros and cons of each method\n",
     " - Using AOI-PMH for harvesting;\n",
     "    + $+$ More efficient harvest:\n",
     "       - faster,\n",
@@ -80,7 +86,7 @@
    "id": "26eba497",
    "metadata": {},
    "source": [
-    "## Example with the OAI-PMH protocol: A python OAI-Harvester"
+    "### Example with the OAI-PMH protocol: A python OAI-Harvester"
    ]
   },
   {
@@ -139,7 +145,7 @@
    "id": "c6a47567",
    "metadata": {},
    "source": [
-    "# Query Zenodo's records through its REST API"
+    "## Zenodo's REST API"
    ]
   },
   {
@@ -195,7 +201,9 @@
    "id": "4cd8011b",
    "metadata": {},
    "source": [
-    "## Example with the `requests` lib - How to recover all ESCAPE2020 community records ?"
+    "### Example with the `requests` lib\n",
+    "\n",
+    "How to recover all ESCAPE2020 community records ?"
    ]
   },
   {
@@ -525,7 +533,7 @@
    "id": "e9007eef",
    "metadata": {},
    "source": [
-    "#### Let's explore a specific ESCAPE2020 entry, for example `agnpy`."
+    "#### A specific ESCAPE2020 entry: `agnpy`."
    ]
   },
   {
diff --git a/examples/notebooks/ossr_api.ipynb b/examples/notebooks/ossr_api.ipynb
index e8c3ab47..51032b39 100644
--- a/examples/notebooks/ossr_api.ipynb
+++ b/examples/notebooks/ossr_api.ipynb
@@ -8,6 +8,14 @@
     "# Find ESCAPE OSSR records"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "5e008b43",
+   "metadata": {},
+   "source": [
+    "## Getting all the records"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -214,7 +222,7 @@
    "id": "c4b08fde",
    "metadata": {},
    "source": [
-    "### Using keywords:"
+    "### Using keywords"
    ]
   },
   {
-- 
GitLab