diff --git a/eossr/api/tests/__init__.py b/eossr/api/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/eossr/api/tests/test_zenodo.py b/eossr/api/tests/test_zenodo.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d84ead5c672274dddf9e0de7b27791a885c0c7a
--- /dev/null
+++ b/eossr/api/tests/test_zenodo.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+import json
+import unittest
+import requests
+from pathlib import Path
+from os import remove
+from os.path import dirname, realpath, join
+from eossr.api.zenodo import ZenodoAPI
+
+ROOT_DIR = dirname(realpath("codemeta.json"))
+
+
+class TestZenodoApiSandbox(unittest.TestCase):
+ def test_initialization_sandbox(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=True,
+ proj_root_dir=ROOT_DIR)
+
+ assert isinstance(z, ZenodoAPI)
+ assert z.zenodo_api_url == 'https://sandbox.zenodo.org/api'
+ assert z.access_token == token
+ assert type(z.exist_codemeta_file) == bool
+ assert type(z.exist_zenodo_metadata_file) == bool
+ assert z.proj_root_dir == Path(ROOT_DIR)
+ assert z.path_codemeta_file == z.proj_root_dir
+ assert z.path_zenodo_metadata_file == z.proj_root_dir
+ assert isinstance(z.proj_root_dir, (Path, str))
+
+
+class TestZenodoAPI(unittest.TestCase):
+ def test_initialization(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=False,
+ proj_root_dir=ROOT_DIR)
+
+ assert isinstance(z, ZenodoAPI)
+ assert z.zenodo_api_url == 'https://zenodo.org/api'
+ assert z.access_token == token
+ assert type(z.exist_codemeta_file) == bool
+ assert type(z.exist_zenodo_metadata_file) == bool
+ assert z.proj_root_dir == Path(ROOT_DIR)
+ assert z.path_codemeta_file == z.proj_root_dir
+ assert z.path_zenodo_metadata_file == z.proj_root_dir
+ assert isinstance(z.proj_root_dir, (Path, str))
+
+ def test_zenodo_api_methods(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=False,
+ proj_root_dir=ROOT_DIR)
+
+ test_id = '42'
+ z.search_codemeta_file()
+ test_filename = join(ROOT_DIR, z.path_codemeta_file)
+ path_test_filename = './'
+
+ fetch_user_entry = z.fetch_user_entries()
+ create_new_entry = z.create_new_entry()
+ fetch_single_entry = z.fetch_entry(
+ entry_id=test_id
+ )
+ # upload_file_entry = z.upload_file_entry(
+ # entry_id=test_id,
+ # name_file=test_filename,
+ # path_file=path_test_filename
+ # )
+ upload_metadata_entry = z.update_metadata_entry(
+ entry_id=test_id,
+ json_metadata=test_filename
+ )
+ erase_entry = z.erase_entry(
+ entry_id=test_id
+ )
+ erase_file_entry = z.erase_file_entry(
+ entry_id=test_id,
+ file_id=test_id
+ )
+ publish_entry = z.publish_entry(
+ entry_id=test_id
+ )
+ new_version_entry = z.new_version_entry(
+ entry_id=test_id
+ )
+ community_entries = z.fetch_community_entries()
+ fetch_ids = z.fetch_community_entries_per_id()
+ fetch_filenames = z.fetch_community_entries_per_title()
+
+ assert isinstance(fetch_user_entry, requests.models.Response)
+ assert isinstance(create_new_entry, requests.models.Response)
+ assert isinstance(fetch_single_entry, requests.models.Response)
+ # assert isinstance(upload_file_entry, requests.models.Response)
+ assert isinstance(upload_metadata_entry, requests.models.Response)
+ assert isinstance(erase_entry, requests.models.Response)
+ assert isinstance(erase_file_entry, requests.models.Response)
+ assert isinstance(publish_entry, requests.models.Response)
+ assert isinstance(new_version_entry, requests.models.Response)
+ assert isinstance(community_entries, requests.models.Response)
+ assert isinstance(fetch_ids, list)
+ assert isinstance(fetch_filenames, list)
+
+ def test_search_codemeta_file(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=False,
+ proj_root_dir=ROOT_DIR)
+
+ assert z.exist_codemeta_file is False
+ z.search_codemeta_file()
+ assert z.exist_codemeta_file is True
+
+ codemeta_file_path = Path(ROOT_DIR) / 'codemeta.json'
+ assert z.path_codemeta_file == codemeta_file_path
+ assert codemeta_file_path.is_file()
+ print(z.path_codemeta_file, type(z.path_codemeta_file))
+ with open(z.path_codemeta_file) as f:
+ json.load(f)
+
+ def test_search_zenodo_json_file(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=False,
+ proj_root_dir=ROOT_DIR)
+
+ assert z.exist_zenodo_metadata_file is False
+ z.search_zenodo_json_file()
+ assert z.exist_zenodo_metadata_file is False
+
+ def test_conversion_codemeta2zenodo_and_search_zenodo_file(self):
+ token = 'FakeToken'
+ z = ZenodoAPI(access_token=token,
+ sandbox=False,
+ proj_root_dir=ROOT_DIR)
+
+ z.search_codemeta_file()
+ z.conversion_codemeta2zenodo()
+
+ z.search_zenodo_json_file()
+ assert z.exist_zenodo_metadata_file is True
+
+ zenodo_file_path = Path(ROOT_DIR) / '.zenodo.json'
+ assert z.path_zenodo_metadata_file == zenodo_file_path
+ assert zenodo_file_path.is_file()
+ with open(z.path_zenodo_metadata_file) as f:
+ json.load(f)
+
+ remove(z.path_zenodo_metadata_file)
diff --git a/eossr/api/zenodo.py b/eossr/api/zenodo.py
new file mode 100644
index 0000000000000000000000000000000000000000..36b0ef88592c5391094119715afd0f6fc567cad5
--- /dev/null
+++ b/eossr/api/zenodo.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python
+
+import sys
+import json
+import pprint
+import requests
+from os.path import abspath
+from pathlib import Path
+from ..metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
+
+
+class ZenodoAPI:
+ def __init__(self, access_token, sandbox=True, proj_root_dir='./'):
+ """
+ Manages the communication with the (sandbox.)zenodo REST API through the Python request library.
+ This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM**
+ the following tasks within the (sandbox.)zenodo api environment:
+
+ - Fetches a user's published entries,
+ - Creates a new deposit,
+ - Fetches any published record,
+ - Creates a new version of an existing deposit,
+ - Uploads files to a specific Zenodo entry,
+ - Erases a non-published entry / new version draft,
+ - Erases (old version) files from an entry (when creating a new_version entry and uploading
+ new_version files),
+ - Uploads information to the entry (Zenodo compulsory deposit information),
+ - Publishes an entry
+ - Find all the published community entries
+ * per title
+ * per entry_id
+
+
+ :param access_token: str
+ Personal access token to (sandbox.)zenodo.org/api
+ :param sandbox: bool
+ Communicates with either zenodo or sandbox.zenodo api
+ """
+
+ if sandbox:
+ zenodo_api_url = "https://sandbox.zenodo.org/api"
+ else:
+ zenodo_api_url = "https://zenodo.org/api"
+
+ self.zenodo_api_url = zenodo_api_url
+ self.access_token = access_token
+ self.parameters = {'access_token': self.access_token}
+
+ self.proj_root_dir = Path(proj_root_dir)
+ self.exist_codemeta_file = False
+ self.path_codemeta_file = self.proj_root_dir
+ self.exist_zenodo_metadata_file = False
+ self.path_zenodo_metadata_file = self.proj_root_dir
+
+ def fetch_user_entries(self):
+ """
+ Fetch the published entries of an user. Works to tests connection to Zenodo too.
+
+ GET method to {zenodo_api_url}/deposit/depositions
+
+ :return: request.get method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions"
+
+ return requests.get(url, params=self.parameters)
+
+ def create_new_entry(self):
+ """
+ Create a new entry / deposition in (sandbox.)zenodo
+
+ POST method to {zenodo_api_url}/deposit/depositions
+
+ :return: request.put method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions"
+ headers = {"Content-Type": "application/json"}
+
+ return requests.post(url, json={}, headers=headers, params=self.parameters)
+
+ def fetch_entry(self, entry_id):
+ """
+ Fetches (recovers all the existing information, as well as links) of an existing Zenodo entry.
+
+ GET method to {zenodo_api_url}/deposit/depositions/{entry_id}
+
+ :param entry_id: str
+ entry_id of the entry to fetch
+
+ :return: request.get method
+ """
+ # In case of entries created by oneself, or entries in the process of being created, the method to fetch
+ # a record is request.get('api/deposit/deposition/{entry_id}') - see also the upload_file_entry method.
+
+ # To fetch any other entry, already published, use:
+ url = f"{self.zenodo_api_url}/records/{entry_id}"
+ return requests.get(url, params=self.parameters)
+
+ def upload_file_entry(self, entry_id, name_file, path_file):
+ """
+ Upload a file to a Zenodo entry. If first retrieve the entry by a GET method to the
+ {zenodo_api_url}/deposit/depositions/{entry_id}.
+
+ PUT method to {bucket_url}/{filename}. The full api url is recovered when the entry is firstly retrieved.
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry
+ :param name_file: str
+ File name of the file when uploaded
+ :param path_file: str
+ Path to the file to be uploaded
+
+ :return: request.put method
+ """
+ # 1 - Retrieve and recover information of a record that is in process of being published
+ fetch = requests.get(f"{self.zenodo_api_url}/deposit/depositions/{entry_id}",
+ params=self.parameters)
+
+ # 2 - Upload the files
+ bucket_url = fetch.json()['links']['bucket'] # full url is recovered from previous GET method
+ url = f"{bucket_url}/{name_file}"
+
+ with open(path_file, 'rb') as upload_file:
+ upload = requests.put(url, data=upload_file, params=self.parameters)
+
+ return upload.json()
+
+ def update_metadata_entry(self, entry_id, json_metadata):
+ """
+ Update an entry resource. Data should be the entry information that will be shown when a deposition is visited
+ at the Zenodo site.
+
+ PUT method to {zenodo_api_url}/deposit/depositions/{entry_id}. `data` MUST be included as json.dump(data)
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry
+ :param json_metadata: object
+ json object containing the metadata (compulsory fields) that are enclosed when a new entry is created.
+
+ :return: request.put method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
+ headers = {"Content-Type": "application/json"}
+
+ # The metadata field is already created, just need to be updated.
+ # Thus the root 'metadata' key need to be kept, to indicate the field to be updated.
+ data = {"metadata": json_metadata}
+
+ return requests.put(url, data=json.dumps(data),
+ headers=headers, params=self.parameters)
+
+ def erase_entry(self, entry_id):
+ """
+ Erase an entry/new version of an entry that HAS NOT BEEN published yet.
+ Any new upload/version will be first saved as 'draft' and not published until confirmation (i.e, requests.post)
+
+ DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}.
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry to be erased
+
+ :return: request.delete method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}"
+ return requests.delete(url, params=self.parameters)
+
+ def erase_file_entry(self, entry_id, file_id):
+ """
+ Erase a file from an entry resource.
+ This method is intended to be used for substitution of files (deletion) within an entry by their correspondent
+ new versions.
+ DELETE method to {zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry
+ :param file_id: str
+ Id of the files stored in Zenodo
+
+ :return: requests.delete method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/files/{file_id}"
+ return requests.delete(url, params=self.parameters)
+
+ def publish_entry(self, entry_id):
+ """
+ Publishes an entry in (sandbox.)zenodo
+ POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry
+
+ :return: requests.put method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/publish"
+ return requests.post(url, params=self.parameters)
+
+ def new_version_entry(self, entry_id):
+ """
+ Creates a new version of AN EXISTING entry resource.
+ POST method to {zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion
+
+ :param entry_id: str
+ deposition_id of the Zenodo entry
+
+ :return: requests.post method
+ """
+ url = f"{self.zenodo_api_url}/deposit/depositions/{entry_id}/actions/newversion"
+ parameters = {'access_token': self.access_token}
+
+ return requests.post(url, params=parameters)
+
+ def fetch_community_entries(self, community_name='escape2020', results_per_query=100):
+ """
+ Query the entries within a community.
+ GET method, previous modification of the query arguments, to {zenodo_api_url}/records
+
+
+ :param community_name: str
+ Community name. DEFAULT='escape2020'
+ :param results_per_query: int
+ Number of entries returned per call to the REST API. DEFAULT=100.
+
+ :return: requests.get method
+ """
+ # https://developers.zenodo.org/#list36
+ update_query_args = {'communities': str(community_name),
+ 'size': int(results_per_query)
+ }
+ self.parameters.update(update_query_args)
+
+ # Full answer
+ # content = requests.post(url, params=self.parameters)
+ # Answer items
+ # content.json().keys()
+ # Stats
+ # content.json()['aggregations']
+ # Total num of entries
+ # content.json()['hits']['total']
+ # Zenodo metadata of each entry
+ # [item['metadata'] for item in content.json()['hits']['hits']]
+
+ return requests.get(f"{self.zenodo_api_url}/records", params=self.parameters)
+
+ def fetch_community_entries_per_id(self, community_name='escape2020', results_per_query=100):
+ """
+ Query the `entries ids` of all the entries within a community
+
+ :param community_name: str
+ Community name. DEFAULT='escape2020'
+ :param results_per_query: int
+ Number of entries returned per call to the REST API. DEFAULT=100.
+
+ :return: list
+ List containing the `id`s of each community entry
+ """
+ return [entry['id'] for entry in
+ self.fetch_community_entries(community_name, results_per_query).json()['hits']['hits']]
+
+ def fetch_community_entries_per_title(self, community_name='escape2020', results_per_query=100):
+ """
+ Query the title of all the entries within a community
+
+ :param community_name: str
+ Community name. DEFAULT='escape2020'
+ :param results_per_query: int
+ Number of entries returned per call to the REST API. DEFAULT=100.
+
+ :return: list
+ List containing the title of each community entry
+ """
+ return [entry['metadata']['title'] for entry in
+ self.fetch_community_entries(community_name, results_per_query).json()['hits']['hits']]
+
+ def search_codemeta_file(self):
+ """Check if a `codemeta.json` files exists in the ROOT directory of the project"""
+
+ # root_dir = find_root_directory()
+ root_dir = self.proj_root_dir
+ print(f'\nProject root directory {abspath(root_dir)}') # DEBUG
+
+ codemeta_file = root_dir / 'codemeta.json'
+
+ if codemeta_file.exists():
+ print("\n * Found codemeta.json file within the project !")
+ self.exist_codemeta_file = True
+ self.path_codemeta_file = codemeta_file
+ else:
+ print("\n ! codemeta.json file NOT found in the root directory of the project !")
+
+ def search_zenodo_json_file(self):
+ """Check if a `.zenodo.json` files exists in the ROOT directory of the project"""
+
+ # root_dir = find_root_directory()
+ root_dir = self.proj_root_dir
+
+ zenodo_metadata_file = root_dir / '.zenodo.json'
+
+ if zenodo_metadata_file.exists():
+ print("\n * Found .zenodo.json file within the project !")
+ self.exist_zenodo_metadata_file = True
+ self.path_zenodo_metadata_file = zenodo_metadata_file
+ else:
+ print("\n ! .zenodo.json file NOT found in the root directory of the project !")
+
+ def conversion_codemeta2zenodo(self):
+ """Perform the codemeta2zenodo conversion if a codemeta.json file is found"""
+
+ if self.exist_codemeta_file:
+ print("\n * Creating a .zenodo.json file from your codemeta.json file...")
+ self.path_zenodo_metadata_file = self.path_codemeta_file.parent / '.zenodo.json'
+
+ parse_codemeta_and_write_zenodo_metadata_file(self.path_codemeta_file,
+ self.path_zenodo_metadata_file)
+ else:
+ pass
+
+ def check_upload_to_zenodo(self):
+ """
+ `Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every
+ stage is the correct one.
+
+ Checks:
+ - The existence of a `.zenodo.json` file in the ROOT dir of the project
+ - If not, it checks if it exists a `codemeta.json` file
+ - If it exists it performs the codemeta2zenodo conversion
+ - If not, it exits the program
+
+ - The communication with Zenodo through its API to verify that:
+ - You can fetch an user entries
+ - You can create a new entry
+ - The provided zenodo metadata can be digested, and not errors appear
+ - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED !
+ """
+ # Search for the codemeta.json and the .zenodo.json files within the project
+ self.search_codemeta_file()
+ self.search_zenodo_json_file()
+
+ if not self.exist_zenodo_metadata_file:
+
+ if self.exist_codemeta_file:
+ self.conversion_codemeta2zenodo()
+ else:
+ print("\n ! NO codemeta.json NOR .zenodo.json file found. "
+ "Please add one to the ROOT directory of your project.")
+ sys.exit(-1)
+
+ print("\n * Using the .zenodo.json file to simulate a new upload to Zenodo... \n")
+
+ # 1 - Test connection
+ print("1 --> Testing communication with Zenodo...")
+
+ test_connection = self.fetch_user_entries()
+ if test_connection.status_code == 200:
+ print(" * Test connection status OK !")
+ else:
+ print(" ! ERROR while testing connection status\n", test_connection.json())
+
+ # 2 - Test new entry
+ print("2 --> Testing the creation of a dummy entry to (sandbox)Zenodo...")
+
+ new_entry = self.create_new_entry()
+ if new_entry.status_code == 201:
+ print(" * Test new entry status OK !")
+ else:
+ print(" ! ERROR while testing the creation of new entry\n", new_entry.json())
+
+ # 3 - Test upload metadata
+ print("3 --> Testing the ingestion of the Zenodo metadata...")
+
+ test_entry_id = new_entry.json()['id']
+ with open(self.path_zenodo_metadata_file) as file:
+ metadata_entry = json.load(file)
+ update_metadata = self.update_metadata_entry(test_entry_id,
+ json_metadata=metadata_entry)
+
+ if update_metadata.status_code == 200:
+ print(" * Update metadata status OK !")
+ pprint.pprint(metadata_entry)
+ else:
+ print(" ! ERROR while testing update of metadata\n",
+ update_metadata.json(), "\n", metadata_entry)
+ print(" ! Erasing dummy test entry...\n")
+ erase_error = self.erase_entry(test_entry_id)
+ if erase_error.status_code != 204:
+ print(f" !! ERROR erasing dummy test entry. Please erase it manually at\n "
+ f"{self.zenodo_api_url[:-4]}/deposit")
+ else:
+ print(" - Done.\n")
+ sys.exit(-1)
+
+ # 4 - Test delete entry
+ print("4 --> Testing the deletion of the dummy entry...")
+
+ delete_test_entry = self.erase_entry(test_entry_id)
+ if delete_test_entry.status_code == 204:
+ print(" * Delete test entry status OK !")
+ else:
+ print(" ! ERROR while deleting test entry\n", delete_test_entry.json())
+
+ print("\n\tYAY ! Successful testing of the connection to Zenodo ! \n\n"
+ "You should not face any trouble when uploading a project to Zenodo - if you followed the "
+ "`OSSR how to publish tutorial`:\n"
+ "\t https://escape2020.pages.in2p3.fr/wp3/ossr-pages/page/contribute/publish_tutorial/#3-add-the-following-code-snippet \n"
+ "In case you do, please contact us !\n")
diff --git a/eossr/scripts/check_connection_zenodo.py b/eossr/scripts/check_connection_zenodo.py
new file mode 100644
index 0000000000000000000000000000000000000000..81ade0b6cef7e5ad303079ba181c7101714b4e8d
--- /dev/null
+++ b/eossr/scripts/check_connection_zenodo.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import argparse
+from distutils.util import strtobool
+from eossr.api.zenodo import ZenodoAPI
+
+
+def main():
+ # Required arguments
+ parser = argparse.ArgumentParser(description="Test the connection to zenodo and all the stages of a new upload.")
+
+ parser.add_argument('--token', '-t', type=str,
+ dest='zenodo_token',
+ help='Personal access token to (sandbox)Zenodo',
+ required=True)
+
+ parser.add_argument('--sandbox', '-s', action='store',
+ type=lambda x: bool(strtobool(x)),
+ dest='sandbox_flag',
+ help='Set the Zenodo environment.'
+ 'If True connects with Zenodo. If False with Sandbox Zenodo',
+ default=False)
+
+ parser.add_argument('--project_dir', '-p', action='store',
+ dest='project_dir',
+ help='Path to the root directory of the directory to be uploaded. '
+ 'DEFAULT; assumed to be on it, i.e., "./"',
+ default='./'
+ )
+
+ args = parser.parse_args()
+
+ zenodo = ZenodoAPI(access_token=args.zenodo_token,
+ sandbox=args.sandbox_flag,
+ proj_root_dir=args.project_dir
+ )
+
+ zenodo.test_upload_to_zenodo()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/eossr/scripts/eossr_upload_new_deposit.py b/eossr/scripts/eossr_upload_new_deposit.py
new file mode 100644
index 0000000000000000000000000000000000000000..c68f3d9594313de0143df6ba21b55c48339ca40e
--- /dev/null
+++ b/eossr/scripts/eossr_upload_new_deposit.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+import os
+import json
+import argparse
+from pathlib import Path
+from distutils.util import strtobool
+from eossr.api.zenodo import ZenodoAPI
+from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
+
+
+def create_zenodo_metadata(metadata_filename, repo_root_dir='./'):
+ """
+ Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
+
+ :param metadata_filename: str
+ path and name to the zenodo metadata json file
+ NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
+
+ :param repo_root_dir: str
+ Path to the project root directory to be uploaded
+ """
+ # root_dir = find_root_directory()
+ root_dir = Path(repo_root_dir)
+ print(f'working dir : {root_dir}') # DEBUG
+
+ files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
+ print(f'JSON files found: \n{files_json}')
+
+ zenodo_metadata_filename = metadata_filename
+ codemeta_file = 'codemeta.json'
+
+ if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
+ print(f"\nCreating {zenodo_metadata_filename} automatically on the CI pipeline.\n")
+ parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
+
+ elif os.path.isfile(zenodo_metadata_filename):
+ print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
+ pass
+
+ else:
+ print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
+ f" created during the CI pipeline."
+ f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
+ exit(-1)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo")
+
+ parser.add_argument('--token', '-t', type=str,
+ dest='zenodo_token',
+ help='Personal access token to (sandbox)Zenodo',
+ required=True)
+
+ parser.add_argument('--sandbox', '-s', action='store',
+ type=lambda x: bool(strtobool(x)),
+ dest='sandbox_flag',
+ help='Set the Zenodo environment.'
+ 'If True connects with Zenodo. If False with Sanbox Zenodo',
+ default=False)
+
+ parser.add_argument('--input-dir', '-i', type=str,
+ dest='input_directory',
+ help='Path to the directory containing the files to upload.'
+ 'ALL files will be uploaded.',
+ required=True)
+
+ args = parser.parse_args()
+
+ zenodo = ZenodoAPI(
+ access_token=args.zenodo_token,
+ sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
+ )
+
+ # 1 - create empty deposit
+ new_entry = zenodo.create_new_entry()
+
+ if new_entry.status_code < 399:
+ deposition_id = new_entry.json()['id']
+ doi = new_entry.json()['metadata']['prereserve_doi']['doi']
+ print(f" * Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}")
+ else:
+ print(f" ! ERROR; the NEW entry COULD NOT be created.")
+ print(new_entry.json())
+
+ # 2 - upload files
+ for file in os.listdir(args.input_directory):
+ full_path_file = args.input_directory + '/' + file
+
+ new_upload = zenodo.upload_file_entry(
+ deposition_id,
+ name_file=file,
+ path_file=full_path_file
+ )
+
+ print(f" * File {file} correctly uploaded !\n", new_upload)
+
+ # 3 - Create the zenodo metadata file from a codemeta.json file
+ zenodo_metadata_filename = '.zenodo.json'
+ create_zenodo_metadata(zenodo_metadata_filename)
+
+ # And upload the repository metadata
+ with open(zenodo_metadata_filename) as json_file:
+ entry_metadata = json.load(json_file)
+
+ # entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
+ update_entry = zenodo.update_metadata_entry(
+ deposition_id,
+ json_metadata=entry_metadata
+ )
+
+ if update_entry.status_code < 399:
+ print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !")
+ else:
+ print(f" ! Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
+ update_entry.json())
+
+ # 4 - publish entry
+ publish = zenodo.publish_entry(deposition_id)
+
+ if publish.status_code == 204:
+ print(" * New deposit correctly published !\n")
+ print(f" * The new doi should look like 10.5281/{deposition_id}. However please")
+ print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **")
+ else:
+ print(f" ! New deposit NOT correctly published ! Status {publish.status_code}\n",
+ publish.json())
+
+
+if __name__ == '__main__':
+ main()
diff --git a/eossr/scripts/eossr_upload_new_version_deposit.py b/eossr/scripts/eossr_upload_new_version_deposit.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb9d170cdbaed703e5aaa2807a06d48324fe4738
--- /dev/null
+++ b/eossr/scripts/eossr_upload_new_version_deposit.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+
+
+import os
+import json
+import argparse
+from pathlib import Path
+from distutils.util import strtobool
+from eossr.api.zenodo import ZenodoAPI
+from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file
+
+
+def create_zenodo_metadata(metadata_filename, repo_root_dir='./'):
+ """
+ Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file
+
+ param metadata_filename: str
+ path and name to the zenodo metada json file
+ NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library.
+ """
+ # root_dir = find_root_directory()
+ root_dir = Path(repo_root_dir)
+
+ files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')]
+ print(f'JSON files : {files_json}')
+
+ zenodo_metadata_filename = metadata_filename
+ codemeta_file = 'codemeta.json'
+
+ if codemeta_file in files_json and zenodo_metadata_filename not in files_json:
+ print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n")
+ parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename)
+
+ elif os.path.isfile(zenodo_metadata_filename):
+ print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n")
+ pass
+
+ else:
+ print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was"
+ f" created during the CI pipeline."
+ f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)")
+ exit(-1)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo")
+
+ parser.add_argument('--token', '-t', type=str,
+ dest='zenodo_token',
+ help='Personal access token to (sandbox)Zenodo',
+ required=True)
+
+ parser.add_argument('--sandbox', '-s', action='store',
+ type=lambda x: bool(strtobool(x)),
+ dest='sandbox_flag',
+ help='Set the Zenodo environment.'
+ 'If True connects with Zenodo. If False with Sandbox Zenodo',
+ default=False)
+
+ parser.add_argument('--input-dir', '-i', type=str,
+ dest='input_directory',
+ help='Path to the directory containing the files to upload.'
+ 'ALL files will be uploaded.',
+ required=True)
+
+ parser.add_argument('--deposit_id', '-id', type=str,
+ dest='deposit_id',
+ help='deposit_id of the deposit that is going to be updated by a new version',
+ required=True)
+
+ args = parser.parse_args()
+
+ zenodo = ZenodoAPI(
+ access_token=args.zenodo_token,
+ sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org
+ )
+
+ # 1 - request a new version of an existing deposit
+ new_version = zenodo.new_version_entry(args.deposit_id)
+
+ if new_version.status_code < 399:
+ print(f" * Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !")
+ else:
+ print(f" ! ERROR; new version of the {args.deposit_id} entry COULD NOT be created.")
+ print(new_version.json())
+
+ new_deposition_id = new_version.json()['links']['latest_draft'].rsplit('/')[-1]
+
+ # PRE-2 - If you DO NOT want to erase the old files, comment the following lines
+ old_files_ids = [file['id'] for file in new_version.json()['files']]
+ for file_id in old_files_ids:
+ zenodo.erase_file_entry(
+ new_deposition_id,
+ file_id
+ )
+
+ # 2 - Upload new version of file(s)
+ for file in os.listdir(args.input_directory):
+ full_path_file = args.input_directory + '/' + file
+
+ new_upload = zenodo.upload_file_entry(
+ new_deposition_id,
+ name_file=file,
+ path_file=full_path_file
+ )
+
+ print(f" * File {file} correctly uploaded !\n", new_upload)
+
+ # 3 - Look for a zenodo metadata file, otherwise try to create one
+ zenodo_metadata_filename = '.zenodo.json'
+ create_zenodo_metadata(zenodo_metadata_filename)
+
+ with open(zenodo_metadata_filename) as json_file:
+ update_entry_metadata = json.load(json_file)
+
+ # update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically.
+ update_entry = zenodo.update_metadata_entry(
+ new_deposition_id,
+ json_metadata=update_entry_metadata
+ )
+
+ if update_entry.status_code < 399:
+ print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !\n")
+ else:
+ print(f" ! Repository information NOT correctly uploaded ! Status {update_entry.status_code}\n",
+ update_entry.json())
+
+ # 4 - publish entry - to publish the entry, uncomment the two lone below
+ publish = zenodo.publish_entry(new_deposition_id)
+
+ if publish.status_code == 204:
+ print(" * New version of the old deposition correctly published !\n")
+ print(f" * Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}")
+ print(f" * The new doi should look like 10.5281/{new_deposition_id}. However please")
+ print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **")
+ else:
+ print(f" ! New deposit NOT correctly published ! Status {publish.status_code}\n",
+ publish.json())
+
+
+if __name__ == '__main__':
+ main()
diff --git a/eossr/scripts/parse_last_release_git.sh b/eossr/scripts/parse_last_release_git.sh
new file mode 100755
index 0000000000000000000000000000000000000000..cf70750df0a454e0a90ac3b56582d3e3b906105c
--- /dev/null
+++ b/eossr/scripts/parse_last_release_git.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+REPOSITORY_NAME="$1"
+REPOSITORY_BASE_URL="$2"
+
+LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'`
+
+if [ -z "$LAST_RELEASE" ]; then
+ echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;"
+ wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip
+ mv $REPOSITORY_NAME-master.zip ./build
+else
+ echo "$LAST_RELEASE tag / release found !"
+ wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip
+ mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build
+fi
diff --git a/eossr/scripts/tests/test_eossr_codemeta2zenodo.py b/eossr/scripts/tests/test_eossr_codemeta2zenodo.py
deleted file mode 100644
index 316b254cbbc7b28e7c75244c75f9c3b231146208..0000000000000000000000000000000000000000
--- a/eossr/scripts/tests/test_eossr_codemeta2zenodo.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import subprocess
-from os.path import dirname, realpath, join
-
-ROOT_DIR = dirname(realpath("codemeta.json"))
-
-
-def run_script(*args):
- result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
-
- if result.returncode != 0:
- raise ValueError(
- f"Running {args[0]} failed with return code {result.returncode}"
- f", output: \n {result.stdout}"
- )
-
-
-def test_codemeta2zenodo():
- run_script("eossr-codemeta2zenodo", "-i", join(ROOT_DIR, "codemeta.json"))
diff --git a/eossr/scripts/tests/test_scripts.py b/eossr/scripts/tests/test_scripts.py
new file mode 100644
index 0000000000000000000000000000000000000000..ead140b080a9fab2035590d0dc3b019f0dce3778
--- /dev/null
+++ b/eossr/scripts/tests/test_scripts.py
@@ -0,0 +1,46 @@
+# Various tests taken/based from
+# https://github.com/cta-observatory/cta-lstchain/blob/master/lstchain/scripts/tests/test_lstchain_scripts.py
+
+import pytest
+import subprocess
+import pkg_resources
+from os.path import dirname, realpath, join
+
+ROOT_DIR = dirname(realpath("codemeta.json"))
+
+
+def find_entry_points(package_name):
+ """from: https://stackoverflow.com/a/47383763/3838691"""
+ entrypoints = [
+ ep.name
+ for ep in pkg_resources.iter_entry_points("console_scripts")
+ if ep.module_name.startswith(package_name)
+ ]
+ return entrypoints
+
+
+ALL_SCRIPTS = find_entry_points("eossr")
+
+
+def run_script(*args):
+ result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+
+ if result.returncode != 0:
+ raise ValueError(
+ f"Running {args[0]} failed with return code {result.returncode}"
+ f", output: \n {result.stdout}"
+ )
+
+
+def test_codemeta2zenodo():
+ run_script("eossr-codemeta2zenodo", "-i", join(ROOT_DIR, "codemeta.json"))
+
+
+def test_parse_last_release_git_bash():
+ run_script("which", "parse_last_release_git.sh")
+
+
+@pytest.mark.parametrize("script", ALL_SCRIPTS)
+def test_help_all_scripts(script):
+ """Test for all scripts if at least the help works"""
+ run_script(script, "--help")
diff --git a/setup.py b/setup.py
index 5a3b1bcce2e80aa72dd9d3bac15d70587c993ed5..bc827db0311d309925b1731af7a3513c9abdd9b4 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,10 @@ import re
from setuptools import setup, find_packages
entry_points = {'console_scripts': [
- 'eossr-codemeta2zenodo = eossr.scripts.eossr_codemeta2zenodo:main'
+ 'eossr-codemeta2zenodo = eossr.scripts.eossr_codemeta2zenodo:main',
+ 'eossr-upload-new-deposit = eossr.scripts.eossr_upload_new_deposit:main',
+ 'eossr-upload-new-version-deposit = eossr.scripts.eossr_upload_new_version_deposit:main',
+ 'eossr-check-connection-zenodo = eossr.scripts.check_connection_zenodo:main'
]
}
@@ -19,10 +22,12 @@ setup(
name='eossr',
version=get_property('__version__', 'eossr'),
description="ESCAPE OSSR library",
- # install_requires=[],
+ install_requires=[
+ "requests"
+ ],
packages=find_packages(),
- # scripts=[],
- # tests_require=['pytest'],
+ scripts=['eossr/scripts/parse_last_release_git.sh'],
+ tests_require=['pytest'],
author='Thomas Vuillaume & Enrique Garcia',
author_email='vuillaume<at>lapp.in2p3.fr',
url='https://gitlab.in2p3.fr/escape2020/wp3/eossr',