Commit 6767b510 authored by Enrique Garcia's avatar Enrique Garcia
Browse files

Codemeta implementation

parent 4a5ce312
# Metadata version - do not change
metadata-version: 0.1
# Mandatory entries
title: template_project_escape
authors:
- Enrique Garcia
- Thomas Vuillaume
contact:
- name: Enrique Garcia
- email: garcia@lapp.in2p3.fr
license: MIT
url: https://gitlab.in2p3.fr/escape2020/wp3/template_project_escape
description: A template project to provide software to ESCAPE.
# Optional entries
doi: 10.5281/zenodo.3572655
keywords: null
type: source
grant: 824064
language: python
hardware:
- machine: local
dependencies:
- python>=3.6
- numpy
os:
- 'linux'
- 'osx-64'
......@@ -35,12 +35,6 @@ build_image:
- mkdir -p build && cp *.simg build
- mkdir -p build && cp Singularity* build
# You can add any other sregistry push commands here, and specify a client
# (and make sure your define the encrypted environment credentials in gitlab
# to push to your storage locations of choice.
# Have a look at https://gitlab.com/singularityhub/gitlab-ci/-/blob/master/.gitlab-ci.yml. Ex:
#- /bin/bash build.sh --uri collection/container --cli registry Singularity
artifacts:
paths:
- build/Singularity.simg
......@@ -61,26 +55,16 @@ deploy_zenodo:
### python, pip and wget are already installed in the container
#- cat /etc/os-release # Debian GNU/Linux 10 (buster)
#- pip3 --version # pip 20.1.1
#- python3 --version # 3.6.11 as th
#- python3 --version # 3.6.11 as the tag of the image specifies
- pip3 install requests
### 2 - Get the last tag/release of the repository
- export REPOSITORY_NAME=template_project_escape
- export REPOSITORY_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME.git
- export LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_URL | tail -n1 | sed 's/.*\///'`
- export REPOSITORY_URL=https://gitlab.in2p3.fr/escape2020/wp3/$REPOSITORY_NAME
### 3 - Download the repository and move it to the build directory
- mkdir -p build
- >
if [ -z "$LAST_RELEASE" ]; then
echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"; \
wget -O $REPOSITORY_NAME-master.zip https://gitlab.in2p3.fr/escape2020/wp3/"$REPOSITORY_NAME"/-/archive/master/"$REPOSITORY_NAME"-master.zip; \
mv $REPOSITORY_NAME-master.zip ./build
else
echo "$LAST_RELEASE tag / release found !"; \
wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip https://gitlab.in2p3.fr/escape2020/wp3/"$REPOSITORY_NAME"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip; \
mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
fi
- /bin/bash .zenodoci/parse_last_release.sh $REPOSITORY_NAME $REPOSITORY_URL
- ls ./build
......@@ -101,10 +85,9 @@ deploy_zenodo:
### 4 - To deploy a NEW VERSION to ZENODO: The deposit_id of the entry to be `new_versioned` MUST be provided.
- >
python3 .zenodoci/upload_new_version_deposit.py
--token $ZENODO_TOKEN
--sandbox_zenodo False
--input-directory ./build
--token $ZENODO_TOKEN
--deposit_id $DEPOSIT_ID_ESCAPE_TEMPLATE
only:
- tags
\ No newline at end of file
- tags
{
"metadata": {
"access_right": "open",
"communities": [
{
"identifier": "escape2020"
}
],
"creators": [
{
"affiliation": "LAPP, CNRS",
"name": "Vuillaume, Thomas",
"orcid": "0000-0002-5686-2078"
},
{
"affiliation": "LAPP, CNRS",
"name": "Garcia, Enrique",
"orcid": "0000-0003-2224-4594"
}
],
"description": "A template repository for the ESCAPE project",
"grants": [
{
"id": "10.13039/501100000780::824064"
}
],
"keywords": [
"template"
],
"language": "eng",
"license": "GPL-3.0+",
"notes": "Release Notes: Documentation and implementation of the last version for CI/CD to Zenodo",
"publication_date": "2019-12-12",
"title": "ESCAPE template project",
"upload_type": "software",
"version": "v2.0"
}
}
\ No newline at end of file
from .zenodoapi import *
from .utils_zenodoci import *
from .upload_new_deposit import *
from .upload_new_version_deposit import *
#!/usr/bin/env bash
REPOSITORY_NAME="$1"
REPOSITORY_BASE_URL="$2"
LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'`
if [ -z "$LAST_RELEASE" ]; then
echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"
wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip
mv $REPOSITORY_NAME-master.zip ./build
else
echo "$LAST_RELEASE tag / release found !"
wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip
mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
fi
{
"metadata": {
"title": "template repository ESCAPE",
"upload_type": "software",
"description": "A template repository for the ESCAPE project",
"creators": [{"name": "Garcia, Enrique",
"affiliation": "LAPP, CNRS"},
{"name": "Vuillaume, Thomas",
"affiliation": "LAPP, CNRS",
"orcid": "0000-0002-5686-2078"}],
"access_right": "open",
"license": "GPL-3.0+",
"communities": [{"identifier": "escape2020"}],
"keywords": [],
"language": "eng",
"notes": "(This will appear as a subtitle)",
"version": "2.0",
"grants": [{"id": "10.13039/501100000780::824064"}]
}
}
\ No newline at end of file
......@@ -4,35 +4,69 @@ import json
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from utils_zenodoci import (parse_codemeta_and_write_zenodo_metadata_file,
find_root_directory
)
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
def create_zenodo_metadata(metadata_filename):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
param metadata_filename: str
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
root_dir = find_root_directory()
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(files_json)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
args = parser.parse_args()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo')
z = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
args = parser.parse_args()
z = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - create empty deposit
new_entry = z.create_new_entry()
......@@ -48,19 +82,27 @@ if __name__ == '__main__':
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(deposition_id,
name_file=file,
path_file=full_path_file)
new_upload = z.upload_file_entry(
deposition_id,
name_file=file,
path_file=full_path_file
)
print(f"File {file} correctly uploaded !\n", new_upload)
# 3 - Upload repository information - that you must have filled before the json file !
with open('.zenodoci/repository_information.json') as json_file:
entry_info = json.load(json_file)
# 3 - Create the zenodo metadata file from a codemeta.json file
zenodo_metadata_filename = '.zenodo.json'
create_zenodo_metadata(zenodo_metadata_filename)
# And upload the repository metadata
with open(zenodo_metadata_filename) as json_file:
entry_metadata = json.load(json_file)
# entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(deposition_id,
data=entry_info)
update_entry = z.update_info_entry(
deposition_id,
data=entry_metadata
)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !")
......
......@@ -4,40 +4,75 @@ import json
import argparse
from distutils.util import strtobool
from zenodoapi import ZenodoAPI
from utils_zenodoci import (parse_codemeta_and_write_zenodo_metadata_file,
find_root_directory
)
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
def create_zenodo_metadata(metadata_filename):
"""
Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
param metadata_filename: str
path and name to the zenodo metada json file
NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
"""
root_dir = find_root_directory()
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
print(files_json)
parser.add_argument('--deposit_id', '-id', type=str,
dest='deposit_id',
help='deposit_id of the deposit that is going to be updated by a new version',
required=True)
zenodo_metadata_filename = metadata_filename
codemeta_file = 'codemeta.json'
args = parser.parse_args()
if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
if __name__ == '__main__':
elif os.path.isfile(zenodo_metadata_filename):
print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
pass
else:
print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
f" created during the CI pipeline."
f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
exit(-1)
z = ZenodoAPI(access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
# Required arguments
parser.add_argument('--token', '-t', type=str,
dest='zenodo_token',
help='Personal access token to (sandbox)Zenodo',
required=True)
parser.add_argument('--sandbox_zenodo', '-s', action='store',
type=lambda x: bool(strtobool(x)),
dest='sandbox_flag',
help='Set the Zenodo environment.'
'If True connects with Zenodo. If False with Sanbox Zenodo',
default=False)
parser.add_argument('--input-directory', '-i', type=str,
dest='input_directory',
help='Path to the directory containing the files to upload.'
'ALL files will be uploaded.',
required=True)
parser.add_argument('--deposit_id', '-id', type=str,
dest='deposit_id',
help='deposit_id of the deposit that is going to be updated by a new version',
required=True)
args = parser.parse_args()
z = ZenodoAPI(
access_token=args.zenodo_token,
sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
)
# 1 - request a new version of an existing deposit
new_version = z.new_version_entry(args.deposit_id)
......@@ -52,27 +87,35 @@ if __name__ == '__main__':
# PRE-2 - If you DO NOT want to erase the old files, comment the following lines
old_files_ids = [file['id'] for file in new_version.json()['files']]
for file_id in old_files_ids:
z.erase_file_entry(new_deposition_id,
file_id)
z.erase_file_entry(
new_deposition_id,
file_id
)
# 2 - Upload new version of file(s)
for file in os.listdir(args.input_directory):
full_path_file = args.input_directory + '/' + file
new_upload = z.upload_file_entry(new_deposition_id,
name_file=file,
path_file=full_path_file)
new_upload = z.upload_file_entry(
new_deposition_id,
name_file=file,
path_file=full_path_file
)
print(f"File {file} correctly uploaded !\n", new_upload)
# 3 - Update metadata info
with open('.zenodoci/repository_information.json') as json_file:
update_entry_info = json.load(json_file)
# 3 - Look for a zenodo metadata file, otherwise try to create one
zenodo_metadata_filename = '.zenodo.json'
create_zenodo_metadata(zenodo_metadata_filename)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
with open(zenodo_metadata_filename) as json_file:
update_entry_metadata = json.load(json_file)
update_entry = z.update_info_entry(new_deposition_id,
data=update_entry_info)
# update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
update_entry = z.update_info_entry(
new_deposition_id,
data=update_entry_metadata
)
if update_entry.status_code < 399:
print(f"Status {update_entry.status_code}. Repository information correctly uploaded !\n")
......
import os
import sys
import json
from pathlib import Path
from distutils.util import strtobool
def parse_person_schema_property(person_property, contributor_field):
"""
Parse the Person Schema property correctly
Parameters:
--------
person_property: dict
dictionary codemeta key with the a list or a single Person property item.
contributor_field : str
contributor type {'editor', 'producer', 'sponsor'} or publisher, although the last one can only happen if
`upload_type` is publication (NOT SUPPORTED - contact E. Garcia by email).
Returns:
--------
zenodo_person: dict
dictionary with the correct zenodo syntax for all {author, contributor, maintainer}.
"""
zenodo_person = {}
special_contributor_cases = ['editor', 'producer', 'publisher', 'provider', 'sponsor']
name = person_property['familyName']
if 'givenName' in person_property:
name += f', {person_property["givenName"]}'
zenodo_person['name'] = name
if "@id" in person_property:
if 'orcid.org/' in person_property["@id"]: # "https://orcid.org/0000-0002-5686-2078" format not accepted
zenodo_person['orcid'] = person_property["@id"].split('orcid.org/')[-1]
else:
zenodo_person['orcid'] = person_property["@id"]
if "affiliation" in person_property:
zenodo_person['affiliation'] = person_property['affiliation']['name']
# Parse correctly the contributors
if contributor_field in special_contributor_cases:
if contributor_field is 'provider' or contributor_field is 'publisher':
zenodo_person['type'] = 'Other'
else:
try:
zenodo_person['type'] = person_property["type"]
except:
zenodo_person['type'] = contributor_field
return zenodo_person
def add_author_metadata(zenodo_file, codemt_file, field):
"""
Aux function to parse correctly all the authors, contributors and maintainers that can be found at the
codemeta.json file
zenodo_file: dict
metadata dictionary with the zenodo syntax
codem_file: list or dict
metadata dictionary key field with the codemeta syntax
field: str
codemeta key field specifying creator {author, contributor, maintainer, creator}, or
contributors {editor, sponsor, producer, project manager...}
"""
full_contacts = {}
creators_fields = ['author', 'creator', 'maintainer', 'contributor']
contributors_fields = ['editor', 'producer', 'publisher', 'provider', 'sponsor']
# First create the full contact agenda by field
if type(codemt_file[field]) is list:
for person_property in codemt_file[field]:
zenodo_person = parse_person_schema_property(person_property, field)
# 'name' is the only key that MUST be contained in a person_property at least
full_contacts[zenodo_person['name']] = zenodo_person
else:
zenodo_person = parse_person_schema_property(codemt_file[field], field)
full_contacts[zenodo_person['name']] = zenodo_person
# then save each person by field and avoid duplicates
for i, person in enumerate(full_contacts):
if field in creators_fields:
# Contributors and maintainers in the same zenodo key
if i == 0 and 'creators' not in zenodo_file:
zenodo_file['creators'] = []
elif person not in zenodo_file['creators']:
zenodo_file['creators'].append(full_contacts[person])
else:
pass # avoid duplicates
elif field in contributors_fields:
if i == 0 and 'contributors' not in zenodo_file:
zenodo_file['contributors'] = []
elif person not in zenodo_file['contributors']:
zenodo_file['contributors'].append(full_contacts[person])
else:
pass # avoid duplicates
def find_matching_metadata(codemeta_json):
"""
Please note that the following fields are ASSUMED. If they are not correct, change them, or contact us otherwise.
"access_right": "open"
"language": "eng"
param codemeta_json: dict
already parsed dictionary containing the metadata of the codemeta.json file
Returns:
--------
metadata_zenodo : dict
dictionary cotaining the metadata information found at the codemeta.json file but written using the Zenodo
syntax.
"""
person_filed = ['author', 'creator', 'maintainer', 'contributor', 'editor', 'producer', 'publisher',
'provider', 'sponsor']
metadata_zenodo = {'language': 'eng',
'access_right': 'open'}
if codemeta_json["@type"] == "SoftwareSourceCode":
metadata_zenodo['upload_type'] = 'software'
else:
metadata_zenodo['upload_type'] = ''
print("\nCould not identify the type of schema in the `codemeta.json file`.\n"
"Thus the 'upload_type' within the `.zenodo.json` file was left EMPTY.\n"
"Please fill it up by yourself - otherwise zenodo will NOT be able to publish your entry.\n")
if 'name' in codemeta_json:
metadata_zenodo['title'] = codemeta_json['name']
if 'description' in codemeta_json:
metadata_zenodo['description'] = codemeta_json['description']
if 'softwareVersion' in codemeta_json and 'version' not in codemeta_json:
metadata_zenodo['version'] = codemeta_json['softwareVersion']
elif 'version' in codemeta_json and 'softwareVersion' not in codemeta_json:
metadata_zenodo['version'] = codemeta_json['version']
else:
metadata_zenodo['version'] = codemeta_json['version']
if 'keywords' in codemeta_json:
if type(codemeta_json['keywords']) == list:
metadata_zenodo['keywords'] = codemeta_json['keywords']
else:
metadata_zenodo['keywords'] = [codemeta_json['keywords']]
if 'license' in codemeta_json:
metadata_zenodo['license'] = codemeta_json['license'].split('/')[-1] # TODO to be improved
if 'releaseNotes' in codemeta_json:
metadata_zenodo['notes'] = "Release Notes: " + codemeta_json['releaseNotes']
if 'citation'