diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 853e44d91aefe6f1468848563778e00fadb4e915..7a4a1264bdaceea4c60a154f2abd8cd118c0c4aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,18 +69,16 @@ build_docker: deploy_zenodo: stage: zenodo - image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v0.2 + image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest dependencies: - build_docker before_script: - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi - - ls ./build\ - # - eossr-upload-new-deposit --token $ZENODO_TOKEN --sandbox False --input-dir ./build - - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_RECORD_ID + - prepare_upload_zenodo.sh $CI_PROJECT_NAME $CI_PROJECT_DIR + - ls ./build + + - eossr-upload-repository -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_RECORD_ID only: - tags diff --git a/eossr/__init__.py b/eossr/__init__.py index 09888577e849d4579b9b7156abf4d421a4950304..52c2eb0c6397d582ba30a75593e294cf8fcf4ce0 100644 --- a/eossr/__init__.py +++ b/eossr/__init__.py @@ -1 +1,4 @@ +from pathlib import Path + +ROOT_DIR = Path(__file__).absolute().parents[1].as_posix() __version__ = "0.2" diff --git a/eossr/api/__init__.py b/eossr/api/__init__.py index c45bcba149df257e7084016e3e60dd28e517be8f..3d4202ed40d96ca15b0ea797f9504a1084e8c6ff 100644 --- a/eossr/api/__init__.py +++ b/eossr/api/__init__.py @@ -1,10 +1,12 @@ +#!/usr/bin/env python + import requests from . import zenodo from .zenodo import Record, get_zenodo_records, zenodo_api_url __all__ = [ 'zenodo', - 'get_ossr_records' + 'get_ossr_records', ] escape_community = 'escape2020' @@ -18,6 +20,8 @@ def get_ossr_records(search='', sandbox=False, **kwargs): :param search: string A string to refine the search in the OSSR. The default will search for all records in the OSSR. + :param sandbox: bool + Indicates the use of sandbox zenodo or not. :param kwargs: Zenodo query arguments. For an exhaustive list, see the query arguments at https://developers.zenodo.org/#list36 Common arguments are: diff --git a/eossr/api/zenodo/__init__.py b/eossr/api/zenodo/__init__.py index 447713e070857a9bd9a1fedb79de44f5e24c9dda..374cdbe8e5e8308a999ed5e46d7331cedfb4736e 100644 --- a/eossr/api/zenodo/__init__.py +++ b/eossr/api/zenodo/__init__.py @@ -4,23 +4,27 @@ import sys import json import pprint import requests +import warnings from pathlib import Path from urllib.parse import urlencode from urllib.request import urlopen -import warnings from bs4 import BeautifulSoup from zipfile import ZipFile -from ...metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file, converter +from eossr import ROOT_DIR from . import http_status +from ...metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file, converter from ...utils import get_codemeta_from_zipurl - __all__ = [ + 'zenodo_api_url', + 'zenodo_sandbox_api_url', 'ZenodoAPI', 'get_record', 'get_zenodo_records', 'http_status', + 'Record', + 'SimilarRecordError' ] @@ -29,11 +33,10 @@ zenodo_sandbox_api_url = "https://sandbox.zenodo.org/api" class ZenodoAPI: - def __init__(self, access_token, sandbox=True, proj_root_dir='./'): + def __init__(self, access_token, sandbox=True, proj_root_dir=Path(ROOT_DIR)): """ Manages the communication with the (sandbox.)zenodo REST API through the Python request library. - This class is **EXCLUSIVELY** developed to be used within a CI/CD pipeline and to **EXCLUSIVELY PERFORM** - the following tasks within the (sandbox.)zenodo api environment: + The client would allow to perform the following tasks within the (sandbox.)zenodo api environment: - Fetches a user's published entries, - Creates a new deposit, @@ -45,10 +48,16 @@ class ZenodoAPI: new_version files), - Uploads information to the entry (Zenodo compulsory deposit information), - Publishes an entry - - Find all the published community entries + - Finds all the published community entries * per title * per entry_id + - Finds all the records of a user (defined by the zenodo token) + - Searches for similar records within all records associated to an user. + Please note that every request.json() answer has been limited to 50 elements. You can set this value + as follows (once ZenodAPI has been inialised, for example): + z = ZenodoApi(token) + z.parameters.update({'size': INTEGER_NUMBER) :param access_token: str Personal access token to (sandbox.)zenodo.org/api @@ -65,6 +74,7 @@ class ZenodoAPI: self.access_token = access_token self.parameters = {'access_token': self.access_token} + self.parameters.setdefault('size', 50) self._root_dir = Path(proj_root_dir) @property @@ -331,41 +341,51 @@ class ZenodoAPI: print("\n ! NO codemeta.json file found. \n" " Please add one to the ROOT directory of your project to ble able to perform the conversion.") - def zip_root_dir(self): + def zip_root_dir(self, zip_filename=None): """ zip the content of `self.root_dir` into `{self.root_dir.name}.zip` :return: zip_filename: path to the zip archive """ # prepare zip archive - zip_filename = self.root_dir.joinpath(f'{self.root_dir.name}.zip') - print(f"Zipping the content of {self.root_dir} into {zip_filename}") + zip_filename = f'{self.root_dir.absolute().name}.zip' if zip_filename is None else zip_filename + print(f" * Zipping the content of {self.root_dir} into {zip_filename}") with ZipFile(zip_filename, 'w') as zo: for file in self.root_dir.iterdir(): zo.write(file) - print(f"zipping done: {zip_filename}") + print(f"Zipping done: {zip_filename}") return zip_filename - - def upload_dir_content(self, record_id=None, metadata=None, erase_previous_files=True, publish=True): + def upload_dir_content(self, directory=None, record_id=None, metadata=None, erase_previous_files=True, + publish=True): """ Package the project root directory as a zip archive and upload it to Zenodo. If a `record_id` is passed, a new version of that record is created. Otherwise a new record is created. + :param directory: Path + If None; `self.root_dir` directory is used. Otherwise the provided path :param record_id: str, int of None + If a record_id is provided, a new version of the record will be created. :param metadata: dict or None - dictionnary of zenodo metadata + dictionary of zenodo metadata if None, the metadata will be read from a `.zenodo.json` file or a `codemeta.json` file in `self.root_dir` + :param erase_previous_files: bool + In case of making a new version of an existing record (`record_id` not None), erase files from the previous + version + :param publish: bool + If true, publish the record. Otherwise, the record is prepared but publication must be done manually. This + is useful to check or discard the record before publication. """ # prepare new record version if record_id is not None: + record = Record.from_id(record_id, sandbox=self.sandbox) record_id = record.last_version_id new_entry = self.new_version_entry(record_id) new_record_id = new_entry.json()['links']['latest_draft'].rsplit('/')[-1] - print(f"Preparing a new version of record {record_id}") + print(f" * Preparing a new version of record {record_id}") # TODO: log if erase_previous_files: old_files_ids = [file['id'] for file in new_entry.json()['files']] @@ -374,38 +394,45 @@ class ZenodoAPI: new_record_id, file_id ) - print(f"file {file_id} erased") + print(f" - file {file_id} erased") else: + new_entry = self.create_new_entry() new_record_id = new_entry.json()['id'] - print(f"Preparing a new record") - print(f"New record id: {new_record_id}") + print(f" * Preparing a new record") + + print(f" * New record id: {new_record_id}") # get metadata if metadata is not None: - print(f"Record metadata based on provided metadata: {metadata}") + print(f" * Record metadata based on provided metadata: {metadata}") elif self.path_zenodo_file.exists(): - print(f"Record metadata based on zenodo file {self.path_codemeta_file}") + print(f" - Record metadata based on zenodo file {self.path_zenodo_file}") with open(self.path_zenodo_file) as file: metadata = json.load(file) - elif self.path_codemeta_file.exists(): - print(f"Record metadata based on codemeta file {self.path_codemeta_file}") + print(f" - Record metadata based on codemeta file {self.path_codemeta_file}") with open(self.path_codemeta_file) as file: codemeta = json.load(file) metadata = converter(codemeta) else: - raise FileNotFoundError("No metadata provided") + raise FileNotFoundError(" ! No metadata file provided") # upload files - for file in self.root_dir.iterdir(): + if directory is None: + dir_to_upload = self.root_dir + else: + dir_to_upload = Path(directory) + + for file in dir_to_upload.iterdir(): self.upload_file_entry(entry_id=new_record_id, name_file=file.name, path_file=file) print(f" * {file.name} uploaded") + # and update metadata self.update_metadata_entry(entry_id=new_record_id, json_metadata=metadata) - print("metadata updated successfully") + print(" * Metadata updated successfully") # publish new record if publish: @@ -494,9 +521,40 @@ class ZenodoAPI: "You should not face any trouble when uploading a project to Zenodo") def get_user_records(self): + """Finds all the records associated with a user (defined by the zenodo token)""" request = self.fetch_user_entries() return [Record(hit) for hit in request.json()] + def find_similar_records(self, record): + """ + Find similar records in the owner records. + This check is not exhaustive and is based only on a limited number of parameters. + + :param record: `eossr.api.zenodo.Record` + :return: list[Record] + list of similar records + """ + similar_records = [] + user_records = self.get_user_records() + for user_rec in user_records: + if user_rec.title == record.title: + similar_records.append(user_rec) + + if 'related_identifiers' in user_rec.data['metadata'] and \ + 'related_identifiers' in record.data['metadata']: + + relid1 = [r['identifier'] for r in user_rec.data['metadata']['related_identifiers']] + relid2 = [r['identifier'] for r in record.data['metadata']['related_identifiers']] + + if set(relid1).intersection(relid2): + similar_records.append(user_rec) + + return similar_records + + +class SimilarRecordError(Exception): + pass + class Record: @@ -543,7 +601,6 @@ class Record: url = Path(self.data['links']['self']).parent.joinpath(str(record_id)) return Record(requests.get(url).json()) - def print_info(self): metadata = self.data['metadata'] descrp = BeautifulSoup(metadata['description'], features='html.parser').get_text() @@ -580,7 +637,7 @@ class Record: @property def doi(self): - if not 'doi' in self.data: + if 'doi' not in self.data: raise KeyError(f"Record {self.id} does not have a doi") return self.data['doi'] @@ -597,6 +654,8 @@ def get_zenodo_records(search='', sandbox=False, **kwargs): :param search: string A string to refine the search in Zenodo. The default will search for all records. + :param sandbox: bool + Indicates the use of sandbox zenodo or not. :param kwargs: Zenodo query arguments. For an exhaustive list, see the query arguments at https://developers.zenodo.org/#list36 Common arguments are: @@ -656,13 +715,17 @@ def get_record(record_id, sandbox=False): """ Get a record from its id - :param record_id: int + :param record_id: int or str + Zenodo record id number. + :param sandbox: bool + Indicates the use of sandbox zenodo or not. + :return: Record """ api_url = zenodo_sandbox_api_url if sandbox else zenodo_api_url url = f"{api_url}/records/{record_id}" - json = requests.get(url).json() - if 'status' in json.keys(): - raise ValueError(f"Error {json['status']} : {json['message']}") + answer = requests.get(url).json() + if 'status' in answer.keys(): + raise ValueError(f"Error {answer['status']} : {answer['message']}") else: - return Record(json) + return Record(answer) diff --git a/eossr/api/zenodo/tests/test_zenodo.py b/eossr/api/zenodo/tests/test_zenodo.py index 54e7f655743d944aa62996bfbad9d4e539814727..2fcc3acb04d98f9160ccb389766f90ff2cf2b835 100644 --- a/eossr/api/zenodo/tests/test_zenodo.py +++ b/eossr/api/zenodo/tests/test_zenodo.py @@ -5,28 +5,29 @@ import shutil import unittest import requests import pytest -from pathlib import Path import tempfile +from pathlib import Path +from eossr import ROOT_DIR from eossr.api.zenodo import ZenodoAPI, get_zenodo_records, get_record, Record -ROOT_DIR = Path('codemeta.json').parent eossr_test_lib_id = 930570 # test library in sandbox (owner: T. Vuillaume) + class TestZenodoApiSandbox(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestZenodoApiSandbox, self).__init__(*args, **kwargs) self.token = 'FakeToken' self.zenodo = ZenodoAPI(access_token=self.token, - sandbox=True, - proj_root_dir=ROOT_DIR) + sandbox=True, + proj_root_dir=ROOT_DIR) def test_initialization_sandbox(self): assert isinstance(self.zenodo, ZenodoAPI) assert self.zenodo.api_url == 'https://sandbox.zenodo.org/api' assert self.zenodo.access_token == self.token - assert self.zenodo.root_dir == ROOT_DIR + assert self.zenodo.root_dir == Path(ROOT_DIR) assert self.zenodo.path_codemeta_file == self.zenodo.root_dir.joinpath('codemeta.json') assert self.zenodo.path_zenodo_file == self.zenodo.root_dir.joinpath('.zenodo.json') assert isinstance(self.zenodo.root_dir, (Path, str)) @@ -58,14 +59,14 @@ class TestZenodoAPINoToken(unittest.TestCase): super(TestZenodoAPINoToken, self).__init__(*args, **kwargs) self.token = 'FakeToken' self.zenodo = ZenodoAPI(access_token=self.token, - sandbox=False, - proj_root_dir=ROOT_DIR) + sandbox=False, + proj_root_dir=ROOT_DIR) def test_initialization(self): assert isinstance(self.zenodo, ZenodoAPI) assert self.zenodo.api_url == 'https://zenodo.org/api' assert self.zenodo.access_token == self.token - assert self.zenodo.root_dir == ROOT_DIR + assert self.zenodo.root_dir == Path(ROOT_DIR) assert isinstance(self.zenodo.root_dir, (Path, str)) @@ -73,7 +74,7 @@ class TestZenodoAPINoToken(unittest.TestCase): class TestZenodoAPIToken(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestZenodoAPIToken, self).__init__(*args, **kwargs) - self.root_dir = ROOT_DIR + self.root_dir = Path(ROOT_DIR) self.token = os.getenv('SANDBOX_ZENODO_TOKEN') self.zenodo = ZenodoAPI(access_token=self.token, sandbox=True, proj_root_dir=ROOT_DIR) @@ -98,7 +99,7 @@ class TestZenodoAPIToken(unittest.TestCase): # prepare upload in a tmpdir with tempfile.TemporaryDirectory() as tmpdirname: print(f"tmpdir {tmpdirname}") - shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmpdirname) + shutil.copy(Path(ROOT_DIR).joinpath('codemeta.json'), tmpdirname) _, filename = tempfile.mkstemp(dir=tmpdirname) Path(filename).write_text('Hello from eossr unit tests') self.zenodo.set_root_dir(tmpdirname) @@ -113,6 +114,12 @@ class TestZenodoAPIToken(unittest.TestCase): self.zenodo.erase_entry(new_record_id) print(f"{new_record_id} created and deleted") + def test_find_similar_records_sandbox(self): + self.zenodo.parameters.update({'size': 100}) + existing_record = Record.from_id(939075, sandbox=True) # One of the copies of eossr + assert len(self.zenodo.find_similar_records(existing_record)) > 0 + not_existing_record = Record.from_id(767507, sandbox=True) # Sandbox E.GARCIA ZenodoCI_vTest Records + assert self.zenodo.find_similar_records(not_existing_record) == [] def test_get_zenodo_records(): @@ -148,16 +155,9 @@ def test_get_record_sandbox(): assert record.data['doi'] == '10.5072/zenodo.520735' -def test_codemeta_in_zip(): - record = get_record(927064, sandbox=True) - codemeta = record.get_codemeta() - assert codemeta['name'] == 'eossr-testlib' - - def test_write_record_zenodo(test_get_record_4923992, tmpdir): record = test_get_record_4923992 record.write_zenodo(filename=tmpdir/'.zenodo.json') with open(tmpdir/'.zenodo.json') as file: json_dict = json.load(file) assert json_dict['conceptdoi'] == '10.5281/zenodo.3572654' - diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index bb5475d3093c43d145ed00820dcb7ad419b55972..e0aab931f183d0b2b7a88d06b2f5b2e14928b2c0 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import json +from pathlib import Path codemeta_creators_fields = ['author', 'creator', 'maintainer', 'contributor'] codemeta_contributors_fields = \ @@ -262,22 +263,28 @@ def converter(codemeta_dict, add_escape2020=True): return converter.zenodo_data -def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, zenodo_outname, add_escape2020=True): +def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, + add_escape2020=True, overwrite=True): """ - Reads the codemeta.json file and creates a new `.zenodo.json` file. This - file will contain the SAME information that in the codemeta.json file but - *** WITH THE ZENODO SYNTAX. *** + Reads the codemeta.json file and creates a new `.zenodo.json` file in outdir. + This file contains the same information that in the codemeta.json file but following the zenodo metadata schema. codemeta_filename: str or Path path to the codemeta.json file - zenodo_outname: str or Path - path and name to the zenodo metada json file - NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored - in the root directory of the library. + outdir: str or Path + path to the outdir where the file `.zenodo.json` will be created + add_escape2020: bool + adds escape2020 metadata in zenodo metadata file + overwrite: bool + overwrite existing `.zendoo.json` file in `outdir` """ converter = CodeMeta2ZenodoController.from_file(codemeta_filename) converter.convert() if add_escape2020: converter.add_escape2020_community() converter.add_escape2020_grant() - converter.write_zenodo(zenodo_outname) + outfile = Path(outdir).joinpath('.zenodo.json') + if not outfile.exists() or overwrite: + converter.write_zenodo(outfile.name) + else: + raise FileExistsError(f"The file {outfile} exists. Use overwrite.") diff --git a/eossr/metadata/tests/test_codemeta2zenodo.py b/eossr/metadata/tests/test_codemeta2zenodo.py index 9772cf3b71288af242921416b195930d14bfa40c..bc8601091c24f6a5af6ad096f227d361479e8535 100644 --- a/eossr/metadata/tests/test_codemeta2zenodo.py +++ b/eossr/metadata/tests/test_codemeta2zenodo.py @@ -1,8 +1,10 @@ import json import unittest import tempfile +import pytest from os.path import dirname, realpath, join from eossr.metadata import codemeta2zenodo +from pathlib import Path SAMPLES_DIR = join(dirname(realpath(__file__)), "samples") ROOT_DIR = dirname(realpath("codemeta.json")) @@ -60,6 +62,13 @@ zenodo_entries = [ ] +@pytest.fixture() +def tmp_dir(tmp_path): + test_dir = tmp_path + test_dir.mkdir(exist_ok=True) + return test_dir + + def test_Codemeta2ZenodoController(): codemeta_file = join(ROOT_DIR, "codemeta.json") converter = codemeta2zenodo.CodeMeta2ZenodoController.from_file(codemeta_file) @@ -120,30 +129,31 @@ def test_parse_person_schema_property(): assert zenodo_metadata['type'] == 'Other' -class TestConverting(unittest.TestCase): - - def test_converter(self): - with open(join(SAMPLES_DIR, "codemeta_sample1.json")) as file: - codemeta = json.load(file) - zenodo = codemeta2zenodo.converter(codemeta) - assert zenodo['communities'][0]['identifier'] == 'escape2020' +# class TestConverting(unittest.TestCase): - def test_sample_file_conversion(self): +def test_converter(): + with open(join(SAMPLES_DIR, "codemeta_sample1.json")) as file: + codemeta = json.load(file) + zenodo = codemeta2zenodo.converter(codemeta) + assert zenodo['communities'][0]['identifier'] == 'escape2020' - outfile = tempfile.NamedTemporaryFile(delete=True) - codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file( - join(SAMPLES_DIR, "codemeta_sample1.json"), outfile.name - ) - json.load(outfile) +def test_sample_file_conversion(tmp_dir): + codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file( + join(SAMPLES_DIR, "codemeta_sample1.json"), + tmp_dir + ) + with open(tmp_dir.joinpath('.zenodo.json').name, 'r') as f: + json.load(f) - def test_root_codemeta_conversion(self): - outfile = tempfile.NamedTemporaryFile(delete=True) - codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file( - join(ROOT_DIR, "codemeta.json"), outfile.name - ) - json.load(outfile) +def test_root_codemeta_conversion(tmp_dir): + codemeta2zenodo.parse_codemeta_and_write_zenodo_metadata_file( + join(ROOT_DIR, "codemeta.json"), + tmp_dir + ) + with open(tmp_dir.joinpath('.zenodo.json').name, 'r') as f: + json.load(f) class TestLicense(unittest.TestCase): diff --git a/eossr/scripts/eossr_codemeta2zenodo.py b/eossr/scripts/eossr_codemeta2zenodo.py index 1970b2c815e6bb04d4066888dd3cf6dfaa1e52f8..434a7e58876ca5903ffe350b924fc0485e5a33f3 100644 --- a/eossr/scripts/eossr_codemeta2zenodo.py +++ b/eossr/scripts/eossr_codemeta2zenodo.py @@ -94,7 +94,7 @@ def main(): # Check overwrite zenodo file if exists if zenodo_metadata_file.exists(): - query_continue( + cont = query_continue( f"\nThe {zenodo_metadata_file.name} file already exists." f"\nIf you continue you will overwrite the file with the metadata found in the {codemeta_file.name} file. " f"\n\nAre you sure ?") @@ -102,7 +102,7 @@ def main(): # Parse the codemeta.json file and create the .zenodo.json file parse_codemeta_and_write_zenodo_metadata_file( codemeta_file, - zenodo_metadata_file + outdir=directory_codemeta ) print("\nConversion codemeta2zenodo done.\n") diff --git a/eossr/scripts/eossr_upload_new_deposit.py b/eossr/scripts/eossr_upload_new_deposit.py deleted file mode 100644 index af1b981128d124b879a135a6c81fdc3a191219ce..0000000000000000000000000000000000000000 --- a/eossr/scripts/eossr_upload_new_deposit.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python - -import os -import json -import argparse -from pathlib import Path -from distutils.util import strtobool -from eossr.api.zenodo import ZenodoAPI -from eossr.api.zenodo.http_status import ZenodoHTTPStatus -from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file - - -def create_zenodo_metadata(metadata_filename, repo_root_dir='./'): - """ - Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file - - :param metadata_filename: str - path and name to the zenodo metadata json file - NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library. - - :param repo_root_dir: str - Path to the project root directory to be uploaded - """ - # root_dir = find_root_directory() - root_dir = Path(repo_root_dir) - print(f'working dir : {root_dir}') # DEBUG - - files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')] - print(f'JSON files found: \n{files_json}') - - zenodo_metadata_filename = metadata_filename - codemeta_file = 'codemeta.json' - - if codemeta_file in files_json and zenodo_metadata_filename not in files_json: - print(f"\nCreating {zenodo_metadata_filename} automatically on the CI pipeline.\n") - parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename) - - elif os.path.isfile(zenodo_metadata_filename): - print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n") - pass - - else: - print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was" - f" created during the CI pipeline." - f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)") - exit(-1) - - -def main(): - parser = argparse.ArgumentParser(description="Upload new deposit entry to Zenodo") - - parser.add_argument('--token', '-t', type=str, - dest='zenodo_token', - help='Personal access token to (sandbox)Zenodo', - required=True) - - parser.add_argument('--sandbox', '-s', action='store', - type=lambda x: bool(strtobool(x)), - dest='sandbox_flag', - help='Set the Zenodo environment.' - 'True to use the sandbox, False (default) to use Zenodo.', - default=False) - - parser.add_argument('--input-dir', '-i', type=str, - dest='input_directory', - help='Path to the directory containing the files to upload.' - 'ALL files will be uploaded.', - required=True) - - args = parser.parse_args() - - zenodo = ZenodoAPI( - access_token=args.zenodo_token, - sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org - ) - - # 1 - create empty deposit - new_entry = zenodo.create_new_entry() - - if new_entry.status_code < 399: - deposition_id = new_entry.json()['id'] - doi = new_entry.json()['metadata']['prereserve_doi']['doi'] - print(f" * Status {new_entry.status_code}. New entry to Zenodo created ! Deposition id {deposition_id}") - else: - print(f" ! ERROR; the NEW entry COULD NOT be created.") - print(new_entry.json()) - - # 2 - upload files - for file in os.listdir(args.input_directory): - full_path_file = args.input_directory + '/' + file - - new_upload = zenodo.upload_file_entry( - deposition_id, - name_file=file, - path_file=full_path_file - ) - - status = ZenodoHTTPStatus(new_upload.status_code, new_upload.json()) - - print(f"{status}\n * File {file} correctly uploaded") - - # 3 - Create the zenodo metadata file from a codemeta.json file - zenodo_metadata_filename = '.zenodo.json' - create_zenodo_metadata(zenodo_metadata_filename) - - # And upload the repository metadata - with open(zenodo_metadata_filename) as json_file: - entry_metadata = json.load(json_file) - - # entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically. - update_entry = zenodo.update_metadata_entry( - deposition_id, - json_metadata=entry_metadata - ) - - status = ZenodoHTTPStatus(update_entry.status_code, update_entry.json()) - print(f" * {status}\nRepository information correctly uploaded !") - - - # 4 - publish entry - publish = zenodo.publish_entry(deposition_id) - - if publish.status_code == 204: - print(" * New deposit correctly published !\n") - print(f" * The new doi should look like 10.5281/{deposition_id}. However please") - print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{deposition_id} **") - else: - print(f" ! New deposit NOT correctly published ! Status {publish.status_code}\n", - publish.json()) - - -if __name__ == '__main__': - main() diff --git a/eossr/scripts/eossr_upload_new_version_deposit.py b/eossr/scripts/eossr_upload_new_version_deposit.py deleted file mode 100644 index 2874a8d686232cf79908336836832ddd9d85d485..0000000000000000000000000000000000000000 --- a/eossr/scripts/eossr_upload_new_version_deposit.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python - - -import os -import json -import argparse -from pathlib import Path -from distutils.util import strtobool -from eossr.api.zenodo import ZenodoAPI -from eossr.api.zenodo.http_status import ZenodoHTTPStatus -from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file - - -def create_zenodo_metadata(metadata_filename, repo_root_dir='./'): - """ - Checks for a zenodo metadata file, otherwise it looks for a codemeta.json file to create a the .zenodo.json file - - param metadata_filename: str - path and name to the zenodo metada json file - NOT TO BE CHANGED. The file must be named `.zenodo.json` and be stored in the root directory of the library. - """ - # root_dir = find_root_directory() - root_dir = Path(repo_root_dir) - - files_json = [file for file in os.listdir(root_dir) if file.endswith('.json')] - print(f'JSON files : {files_json}') - - zenodo_metadata_filename = metadata_filename - codemeta_file = 'codemeta.json' - - if codemeta_file in files_json and zenodo_metadata_filename not in files_json: - print(f"\nCreating {zenodo_metadata_filename} automatically at the CI pipeline.\n") - parse_codemeta_and_write_zenodo_metadata_file(codemeta_file, zenodo_metadata_filename) - - elif os.path.isfile(zenodo_metadata_filename): - print(f"\n{zenodo_metadata_filename} metadata file found in the root directory of the library ! \n") - pass - - else: - print(f"\n{codemeta_file} not found, thus any zenodo_metadata file `{zenodo_metadata_filename}` was" - f" created during the CI pipeline." - f"Please provide one so that the CI can run correctly (examples in the 'codemeta_utils' directory)") - exit(-1) - - -def main(): - parser = argparse.ArgumentParser(description="Upload a new version of an existing deposit to Zenodo") - - parser.add_argument('--token', '-t', type=str, - dest='zenodo_token', - help='Personal access token to (sandbox)Zenodo', - required=True) - - parser.add_argument('--sandbox', '-s', action='store', - type=lambda x: bool(strtobool(x)), - dest='sandbox_flag', - help='Set the Zenodo environment.' - 'True to use the sandbox, False (default) to use Zenodo.', - default=False) - - parser.add_argument('--input-dir', '-i', type=str, - dest='input_directory', - help='Path to the directory containing the files to upload.' - 'ALL files will be uploaded.', - required=True) - - parser.add_argument('--deposit_id', '-id', type=str, - dest='deposit_id', - help='deposit_id of the deposit that is going to be updated by a new version', - required=True) - - args = parser.parse_args() - - zenodo = ZenodoAPI( - access_token=args.zenodo_token, - sandbox=args.sandbox_flag # True for sandbox.zenodo.org !! False for zenodo.org - ) - - # 1 - request a new version of an existing deposit - new_version = zenodo.new_version_entry(args.deposit_id) - - if new_version.status_code < 399: - print(f" * Status {new_version.status_code}. New version of the {args.deposit_id} entry correctly created !") - else: - print(f" ! ERROR; new version of the {args.deposit_id} entry COULD NOT be created.") - print(new_version.json()) - - new_deposition_id = new_version.json()['links']['latest_draft'].rsplit('/')[-1] - - # PRE-2 - If you DO NOT want to erase the old files, comment the following lines - old_files_ids = [file['id'] for file in new_version.json()['files']] - for file_id in old_files_ids: - zenodo.erase_file_entry( - new_deposition_id, - file_id - ) - - # 2 - Upload new version of file(s) - for file in os.listdir(args.input_directory): - full_path_file = args.input_directory + '/' + file - - new_upload = zenodo.upload_file_entry( - new_deposition_id, - name_file=file, - path_file=full_path_file - ) - - status = ZenodoHTTPStatus(new_upload.status_code, new_upload.json()) - print(f"{status}\n * File {file} correctly uploaded") - - # 3 - Look for a zenodo metadata file, otherwise try to create one - zenodo_metadata_filename = '.zenodo.json' - create_zenodo_metadata(zenodo_metadata_filename) - - with open(zenodo_metadata_filename) as json_file: - update_entry_metadata = json.load(json_file) - - # update_entry_info['metadata']['doi'] = doi # In the new version of the API the doi is updated automatically. - update_entry = zenodo.update_metadata_entry( - new_deposition_id, - json_metadata=update_entry_metadata - ) - - status = ZenodoHTTPStatus(update_entry['status_code'], update_entry) - - if not status.is_error(): - print(f" * Status {update_entry.status_code}. Repository information correctly uploaded !\n") - - - # 4 - publish entry - to publish the entry, uncomment the two lone below - publish = zenodo.publish_entry(new_deposition_id) - - if publish.status_code == 204: - print(" * New version of the old deposition correctly published !\n") - print(f" * Old deposition id {args.deposit_id}, new deposition id {new_deposition_id}") - print(f" * The new doi should look like 10.5281/{new_deposition_id}. However please") - print(f" ** Check the upload at {zenodo.zenodo_api_url[:-4]}/deposit/{new_deposition_id} **") - else: - print(f" ! New deposit NOT correctly published ! Status {publish.status_code}\n", - publish.json()) - - -if __name__ == '__main__': - main() diff --git a/eossr/scripts/eossr_upload_repository.py b/eossr/scripts/eossr_upload_repository.py new file mode 100644 index 0000000000000000000000000000000000000000..31c1b65704c5171a16996908c65b0a7d6a690162 --- /dev/null +++ b/eossr/scripts/eossr_upload_repository.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# This script is meant to be used by the continuous integration of a git repository to upload its content to the OSSR + + +import shutil +import argparse +import json +from pathlib import Path +from distutils.util import strtobool +from eossr.api.zenodo import ZenodoAPI, Record, SimilarRecordError +from eossr.metadata.codemeta2zenodo import converter +from eossr.utils import zip_repository + + +def upload(zenodo_token, sandbox_flag, upload_directory, record_id=None, zip_root_dir=False, erase_previous_files=True, + force_new_record=False, publish=True): + """ + Prepares the upload of the content of `upload_directory` to the OSSR. + There must be a metadata file present in the directory to be uploaded. + + It first searches for similar record uploaded by the same Zenodo user. + The codemeta.json file will be copied to the `upload_directory`. + Uploads the content of `upload_directory` using the ZenodoAPI. + + :param zenodo_token: str + Personal access token to the (sandbox.)zenodo.org/api + :param sandbox_flag: bold + Set the Zenodo environment. True to use the sandbox, False to use Zenodo. + :param upload_directory: str or Path + Path to the directory whose content will be uploaded to the OSSR. + :param record_id: int or str + Zenodo record-id of the record that is going to be updated. + If no record_id is provided, a new record will be created in the OSSR. + :param zip_root_dir: bool + If True, the content of the root dir of the repository will be zipped and moved to the `upload_directory` to + be uploaded to the OSSR. + :param erase_previous_files: bool + If True (default), it will erase the files of previous versions of the record before creating and updating the + new version of the record. If False, it will not erase any file and old files will be included in the + new version. + :param force_new_record: bool + If False (default), a new version of the `record_id` record will be created. + If True, a new record - despite that it might already exists one - will be created. + :param publish: bool + If true, publish the record. Otherwise, the record is prepared but publication must be done manually. This + is useful to check or discard the record before publication. + + :return: The `record_id` of the record created/uploaded + `ZenodoAPI.upload_dir_content` answer + """ + + zenodo = ZenodoAPI(access_token=zenodo_token, sandbox=sandbox_flag, proj_root_dir=upload_directory) + + # Loads the metadata files if exists + if zenodo.path_zenodo_file.exists(): + print(f"Record metadata based on zenodo file {zenodo.path_zenodo_file}") + with open(zenodo.path_zenodo_file) as file: + metadata = json.load(file) + elif zenodo.path_codemeta_file.exists(): + print(f"Record metadata based on codemeta file {zenodo.path_codemeta_file}") + with open(zenodo.path_codemeta_file) as file: + codemeta = json.load(file) + metadata = converter(codemeta) + else: + raise FileNotFoundError("No metadata provided") + + metadata_for_check = {'metadata': metadata} + metadata_for_check['id'] = 1 # fake id to create fake record + record = Record(metadata_for_check) + + # Searches for similar records + similar_records = zenodo.find_similar_records(record) + if similar_records and not force_new_record and not record_id: + raise SimilarRecordError( + f"There are similar records in your own records: {similar_records}." + "If you want to update an existing record, provide its record id to make a new version." + "If you still want to make a new record, use --force-new-record.") + + # Zips the root directory content and moves the file to the `input dir` for a later upload + if zip_root_dir: + repo_zipped = zip_repository(zenodo.root_dir, zip_filename=f'{zenodo.root_dir.name}') + shutil.move(repo_zipped, upload_directory) + + # If a codemeta.json file exists and is not present in the upload dir, copy it into this dir. + if not Path(upload_directory).joinpath('codemeta.json').exists() and \ + zenodo.path_codemeta_file.exists(): + shutil.copy(zenodo.path_codemeta_file, upload_directory) + + return zenodo.upload_dir_content(upload_directory, + record_id=record_id, + metadata=metadata, + erase_previous_files=erase_previous_files, + publish=publish + ) + + +def main(): + parser = argparse.ArgumentParser(description="Upload a directory to the OSSR as record." + "The directory must include a valid zenodo or codemeta file to be used" + "as metadata source for the upload." + "If not record_id is passed, a new record is created." + "Otherwise, a new version of the existing record is created." + ) + + parser.add_argument('--token', '-t', type=str, + dest='zenodo_token', + help='Personal access token to (sandbox)Zenodo', + required=True) + + parser.add_argument('--sandbox', '-s', action='store', + type=lambda x: bool(strtobool(x)), + dest='sandbox_flag', + help='Set the Zenodo environment.' + 'True to use the sandbox, False (default) to use Zenodo.', + default=False) + + parser.add_argument('--input-dir', '-i', type=str, + dest='input_directory', + help='Path to the directory containing the files to upload.' + 'All files will be uploaded.', + required=True) + + parser.add_argument('--record_id', '-id', type=str, + dest='record_id', + help='record_id of the deposit that is going to be updated by a new version', + default=None, + required=False) + + parser.add_argument('--force-new-record', + action='store_true', + dest='force_new_record', + help='Force the upload of a new record in case a similar record is found ' + 'in the user existing ones', + ) + + parser.add_argument('--no-publish', + action='store_false', + dest='publish', + help='Optional tag to specify if the record will NOT be published. ' + 'Useful for checking the record before publication of for CI purposes.', + ) + + args = parser.parse_args() + + new_record_id = upload(args.zenodo_token, + args.sandbox_flag, + args.input_directory, + record_id=args.record_id, + force_new_record=args.force_new_record, + publish=args.publish) + + return new_record_id + + +if __name__ == '__main__': + main() diff --git a/eossr/scripts/gitlab_prepare_upload_zenodo.sh b/eossr/scripts/gitlab_prepare_upload_zenodo.sh new file mode 100755 index 0000000000000000000000000000000000000000..fe8753eb7c6e6d23b6d201637f9c17bd89f44bd2 --- /dev/null +++ b/eossr/scripts/gitlab_prepare_upload_zenodo.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +REPOSITORY_NAME="$1" +REPOSITORY_ROOT_DIR="$2" +REPOSITORY_URL=`git config --get remote.origin.url` +LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_URL | tail -n1 | sed 's/.*\///'` + +mkdir -p ./build + +if [ -z "$LAST_RELEASE" ]; then + echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;" + zip-repository -d $REPOSITORY_ROOT_DIR -n $REPOSITORY_NAME + mv $REPOSITORY_NAME.zip ./build +else + echo "$LAST_RELEASE tag / release found !" + zip-repository -d $REPOSITORY_ROOT_DIR -n $CI_PROJECT_NAME-$LAST_RELEASE + mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build +fi + +if [[ -f ./codemeta.json ]]; then + cp ./codemeta.json ./build +fi \ No newline at end of file diff --git a/eossr/scripts/parse_last_release_git.sh b/eossr/scripts/parse_last_release_git.sh deleted file mode 100755 index cf70750df0a454e0a90ac3b56582d3e3b906105c..0000000000000000000000000000000000000000 --- a/eossr/scripts/parse_last_release_git.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -REPOSITORY_NAME="$1" -REPOSITORY_BASE_URL="$2" - -LAST_RELEASE=`git ls-remote --tags --refs --sort="v:refname" $REPOSITORY_BASE_URL.git | tail -n1 | sed 's/.*\///'` - -if [ -z "$LAST_RELEASE" ]; then - echo "No tag / new release found ! - Or error when parsing. Downloading last commit to the repository (master branch) ;" - wget -O $REPOSITORY_NAME-master.zip "$REPOSITORY_BASE_URL"/-/archive/master/"$REPOSITORY_NAME"-master.zip - mv $REPOSITORY_NAME-master.zip ./build -else - echo "$LAST_RELEASE tag / release found !" - wget -O $REPOSITORY_NAME-$LAST_RELEASE.zip "$REPOSITORY_BASE_URL"/-/archive/"$LAST_RELEASE"/"$REPOSITORY_NAME"-"$LAST_RELEASE".zip - mv $REPOSITORY_NAME-$LAST_RELEASE.zip ./build -fi diff --git a/eossr/scripts/tests/test_scripts.py b/eossr/scripts/tests/test_scripts.py index ead140b080a9fab2035590d0dc3b019f0dce3778..37da247046c310ced4405e74efabedb0a9c4a144 100644 --- a/eossr/scripts/tests/test_scripts.py +++ b/eossr/scripts/tests/test_scripts.py @@ -4,7 +4,11 @@ import pytest import subprocess import pkg_resources +import os +import shutil +from pathlib import Path from os.path import dirname, realpath, join +from eossr.scripts import eossr_upload_repository ROOT_DIR = dirname(realpath("codemeta.json")) @@ -33,14 +37,38 @@ def run_script(*args): def test_codemeta2zenodo(): + existing_zenodo_file = Path(ROOT_DIR).joinpath('.zenodo.json') + if existing_zenodo_file.exists(): + existing_zenodo_file.unlink() + run_script("eossr-codemeta2zenodo", "-i", join(ROOT_DIR, "codemeta.json")) + existing_zenodo_file.unlink() def test_parse_last_release_git_bash(): - run_script("which", "parse_last_release_git.sh") + run_script("which", "gitlab_prepare_upload_zenodo.sh") @pytest.mark.parametrize("script", ALL_SCRIPTS) def test_help_all_scripts(script): """Test for all scripts if at least the help works""" run_script(script, "--help") + + +def test_eossr_upload_repository(tmpdir): + path_test_filename = Path(tmpdir).joinpath('test.txt') + Path(path_test_filename).write_text('Hello World') + shutil.copy(Path(ROOT_DIR).joinpath('codemeta.json'), tmpdir) + record_id = eossr_upload_repository.upload(zenodo_token=os.getenv('SANDBOX_ZENODO_TOKEN'), + sandbox_flag=True, + upload_directory=tmpdir, + force_new_record=True + ) + + Path(path_test_filename).write_text('Hello World 2') + eossr_upload_repository.upload(zenodo_token=os.getenv('SANDBOX_ZENODO_TOKEN'), + sandbox_flag=True, + upload_directory=tmpdir, + record_id=record_id + ) + diff --git a/eossr/scripts/zip_repository.py b/eossr/scripts/zip_repository.py new file mode 100644 index 0000000000000000000000000000000000000000..13af8cc4aa7d3c2f5673d1b0141466c49b7dc148 --- /dev/null +++ b/eossr/scripts/zip_repository.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import argparse +from eossr.utils import zip_repository + + +def main(): + parser = argparse.ArgumentParser( + description="Zip the content of a directory (a git project is expected)." + ) + + parser.add_argument('--directory', '-d', type=str, + dest='directory', + help='Path to the directory to be zipped.', + required=True) + + parser.add_argument('--name_zip', '-n', type=str, + dest='name_zip', + help='Zip filename. DEFAULT: directory (basename)', + default=None + ) + + args = parser.parse_args() + zip_repository(args.directory, + args.name_zip, + ) + + +if __name__ == '__main__': + main() diff --git a/eossr/utils.py b/eossr/utils.py index 0ed8be7ab889a530e5137da0fed52f37713d3d5d..0ee7f72fc5ad2d3f113dcc8c1b288543a5f04bb5 100644 --- a/eossr/utils.py +++ b/eossr/utils.py @@ -7,8 +7,10 @@ from pathlib import Path __all__ = [ 'ZipUrl', 'get_codemeta_from_zipurl', + 'zip_repository', ] + class ZipUrl: def __init__(self, url): @@ -58,3 +60,29 @@ def get_codemeta_from_zipurl(url): return codemeta + +def zip_repository(directory, zip_filename=None): + """ + Zip the content of `directory` into `./zip_filename.zip`. + + :param directory: str or Path + Path to the directory to be zipped + :param zip_filename: str + Zip filename name, used to name the zip file. If None, the zip will be named as the directory provided. + + :return: zip_filename: path to the zip archive + """ + # prepare zip archive + directory = Path(directory) + zip_filename = f'{directory.absolute().name}.zip' if zip_filename is None \ + else f'{zip_filename.replace(".zip", "")}.zip' + print(f" * Zipping the content of {directory.name} into {zip_filename}") + + exclude = [zip_filename, '.git'] + files = [f for f in directory.iterdir() if f.name not in exclude] + + with ZipFile(zip_filename, 'w') as zo: + for file in files: + zo.write(file) + print(f"Zipping done: {zip_filename}") + return zip_filename diff --git a/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md b/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md deleted file mode 100644 index f9079db993d2650224d48f99fe111cdead7602ba..0000000000000000000000000000000000000000 --- a/examples/CI_code_snippets/3.ex_CI_upload_new_deposit.md +++ /dev/null @@ -1,54 +0,0 @@ -# Upload a new entry to the OSSR - - - Uses the GitLab CI to upload the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo community). - - Note that the CI will be only triggered with the creation of a new release. - - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as - a separate file. - - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check - that the released code will be successfully uploaded. - - -### Upload to Zenodo -```yaml -stages: - - deploy - -deploy_zenodo: - stage: deploy - image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest - before_script: - - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR - script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi - - ls ./build - - - eossr-upload-new-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID - only: - - tags - -``` - - -### Upload to Sandbox Zenodo -```yaml -stages: - - deploy - -deploy_zenodo: - stage: deploy - image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest - before_script: - - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR - script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi - - ls ./build - - - eossr-upload-new-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $ZENODO_PROJECT_ID - only: - - tags - -``` diff --git a/examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md b/examples/CI_code_snippets/3.ex_CI_upload_ossr.md similarity index 57% rename from examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md rename to examples/CI_code_snippets/3.ex_CI_upload_ossr.md index cfb89d9316c626b67643c177a6e33bc389353b2e..11790735bc1cc7a07194c4f9876b6ff06a70464c 100644 --- a/examples/CI_code_snippets/4.ex_CI_upload_new_version_deposit.md +++ b/examples/CI_code_snippets/3.ex_CI_upload_ossr.md @@ -1,18 +1,21 @@ -# Upload a new version of an existing entry to the OSSR +# Upload to the OSSR - - Uses the GitLab CI to upload a **new version** of the current project to the ESCAPE-OSSR (The ESCAPE2020 Zenodo - community). - - Note that the CI will be only triggered with the creation of a new release. - - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the Zenodo entry as + - Uses the GitLab CI to upload the current project to the ESCAPE OSSR (The ESCAPE2020 Zenodo community). + - Note that the CI will be only triggered with the creation of a new release. + - The `codemeta.json` file - compulsory if you want to run this code - will be also added to the record as a separate file. - The `eossr-check-connection-zenodo` stage will create a fist dummy upload (that will be always erased), to check that the released code will be successfully uploaded. - -**NOTE**. -You should have saved the `deposit_id` of your project as a GitLab environment variable before the CI runs this stage. -To do so: -1. Go to https://zenodo.org/deposit, +## Important note + +The first time CI is run, remove `-id $ZENODO_PROJECT_ID` to create a new record. +After that, **you need to create the CI variable** `$ZENODO_PROJECT_ID` (or `$SANDBOX_ZENODO_PROJECT_ID`) +using the newly created record ID. +**If you don't,** the next release will create a new record instead of updating the existing one. + +To create the CI `$(SANDBOX_)ZENODO_PROJECT_ID` variable: +1. Go to https://zenodo.org/deposit or https://sandbox.zenodo.org/deposit; 2. Click onto your just uploaded project, 3. From your browser search bar, **just** copy the number (your `deposit id`) that it is included in the https direction. - ex: `https://zenodo.org/record/3884963` --> just copy `3884963`. @@ -34,12 +37,10 @@ deploy_zenodo: before_script: - eossr-check-connection-zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - prepare_upload_zenodo.sh $CI_PROJECT_NAME $CI_PROJECT_DIR - ls ./build - - eossr-upload-new-version-deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + - eossr-upload-repository -t $ZENODO_TOKEN -s False -i ./build [-id $ZENODO_PROJECT_ID] only: - tags @@ -57,12 +58,10 @@ deploy_zenodo: before_script: - eossr-check-connection-zenodo --token $SANDBOX_ZENODO_TOKEN --sandbox True -p $CI_PROJECT_DIR script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - prepare_upload_zenodo.sh $CI_PROJECT_NAME $CI_PROJECT_DIR - ls ./build - - eossr-upload-new-version-deposit -t $SANDBOX_ZENODO_TOKEN -s True -i ./build -id $SANDBOX_ZENODO_PROJECT_ID + - eossr-upload-repository -t $SANDBOX_ZENODO_TOKEN -s True -i ./build [-id $SANDBOX_ZENODO_PROJECT_ID] only: - tags diff --git a/examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md b/examples/CI_code_snippets/4.ex_CI_build_image_and_upload_OSSR.md similarity index 86% rename from examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md rename to examples/CI_code_snippets/4.ex_CI_build_image_and_upload_OSSR.md index 46ced01e4bafe552fdef3c99e14bd2b3b8f7013d..8ae5ae9cbe9ce8953f08104d30e61d87e12b3952 100644 --- a/examples/CI_code_snippets/5.ex_CI_build_image_and_upload_OSSR.md +++ b/examples/CI_code_snippets/4.ex_CI_build_image_and_upload_OSSR.md @@ -8,7 +8,7 @@ This code snippet will: - Uploads the next released version of the current project, together with both images, to the ESCAPE-OSSR (The ESCAPE2020 Zenodo community). -Have a look before to the examples in this same directory. +For details on each CI stage, have a look before to the examples in this same directory. ```yaml stages: @@ -16,7 +16,7 @@ stages: - deploy build_singularity_image: - stage: build + stage: build_container image: singularityware/singularity:gitlab-2.6 script: # You should have added before your Singularity recipe in a Singularity dir @@ -32,7 +32,7 @@ build_singularity_image: - tags build_docker_image: - stage: build + stage: build_container image: docker:19.03.12 services: - docker:19.03.12-dind @@ -69,12 +69,10 @@ deploy_zenodo: before_script: - test_connection_zenodo --token $ZENODO_TOKEN --sandbox False -p $CI_PROJECT_DIR script: - - mkdir -p build - - parse_last_release_git.sh $CI_PROJECT_NAME $CI_PROJECT_URL - - if [[ -f ./codemeta.json ]]; then cp ./codemeta.json ./build; fi + - prepare_upload_zenodo.sh $CI_PROJECT_NAME $CI_PROJECT_DIR - ls ./build - - upload_new_version_deposit -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_PROJECT_ID + - eossr-upload-repository -t $ZENODO_TOKEN -s False -i ./build -id $ZENODO_RECORD_ID only: - tags ``` diff --git a/setup.py b/setup.py index f62ace101b7b753872c7f1440d4bf67bc47b305b..c0155cd854c7964b10e76aa51ac1dd7bdfe98c0e 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,10 @@ import re from setuptools import setup, find_packages entry_points = {'console_scripts': [ + 'eoosr-zip-repository = eossr.scripts.zip_repository:main', 'eossr-codemeta2zenodo = eossr.scripts.eossr_codemeta2zenodo:main', - 'eossr-upload-new-deposit = eossr.scripts.eossr_upload_new_deposit:main', - 'eossr-upload-new-version-deposit = eossr.scripts.eossr_upload_new_version_deposit:main', - 'eossr-check-connection-zenodo = eossr.scripts.check_connection_zenodo:main' + 'eossr-upload-repository = eossr.scripts.eossr_upload_repository:main', + 'eossr-check-connection-zenodo = eossr.scripts.check_connection_zenodo:main', ] } @@ -27,7 +27,7 @@ setup( "bs4", ], packages=find_packages(), - scripts=['eossr/scripts/parse_last_release_git.sh'], + scripts=['eossr/scripts/gitlab_prepare_upload_zenodo.sh'], tests_require=['pytest'], author='Thomas Vuillaume & Enrique Garcia', author_email='vuillaume@lapp.in2p3.fr',