From dbe04e726d6e96447a279af0070600579b4af707 Mon Sep 17 00:00:00 2001
From: Pierre Aubert <pierre.aubert.lapp@gmail.com>
Date: Thu, 25 Aug 2022 18:38:50 +0200
Subject: [PATCH] Add scripts to make automatic release within CI
---
.gitlab-ci.yml | 240 ++++++++++++++++++++------------
cmake/createReleaseCurl.py | 135 ++++++++++++++++++
cmake/phoenix_createPackages.sh | 110 +++++++++++++++
3 files changed, 396 insertions(+), 89 deletions(-)
create mode 100644 cmake/createReleaseCurl.py
create mode 100755 cmake/phoenix_createPackages.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 559502c..9eed70e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,3 @@
-image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
stages:
- BuildTestinstall
@@ -6,103 +5,166 @@ stages:
- deploy
dailyBuildMasterAll:
- #cache:
- #untracked: true
- #paths:
- #- build/
-
- stage: BuildTestinstall
- script:
- - export LD_LIBRARY_PATH=/usr/lib
- - env
- - mkdir -p build
- - cd build
- - >
- cmake ..
- -DCMAKE_INSTALL_PREFIX=/usr
- -DSELF_TESTS_MODE=yes
- - make all
- - make install
- - make test
- only:
- - branches
- - tags
- tags:
+ #cache:
+ #untracked: true
+ #paths:
+ #- build/
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+ stage: BuildTestinstall
+ script:
+ - export LD_LIBRARY_PATH=/usr/lib
+ - env
+ - mkdir -p build
+ - cd build
+ - >
+ cmake ..
+ -DCMAKE_INSTALL_PREFIX=/usr
+ -DSELF_TESTS_MODE=yes
+ - make all
+ - make install
+ - make test
+ only:
+ - branches
+ - tags
+ tags:
TestCoverage:
- stage: DocCoverage
- script:
- - export LD_LIBRARY_PATH=/usr/lib
- - env
- - mkdir -p build coverage
- - cd build
- - >
- cmake ..
- -DCMAKE_INSTALL_PREFIX=/usr
- -DSELF_TESTS_MODE=yes
- -DCMAKE_BUILD_TYPE=Coverage
- - make all
- - make install
- - make test
- - make run_all
- - make ExperimentalCoverage
- - cd ../coverage
- - gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
- - gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
- - gcovr -r ../ --exclude-throw-branches
- artifacts:
- paths:
- - coverage
- expire_in: 8d
- only:
- - branches
- - tags
- tags:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+ stage: DocCoverage
+ script:
+ - export LD_LIBRARY_PATH=/usr/lib
+ - env
+ - mkdir -p build coverage
+ - cd build
+ - >
+ cmake ..
+ -DCMAKE_INSTALL_PREFIX=/usr
+ -DSELF_TESTS_MODE=yes
+ -DCMAKE_BUILD_TYPE=Coverage
+ - make all
+ - make install
+ - make test
+ - make run_all
+ - make ExperimentalCoverage
+ - cd ../coverage
+ - gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
+ - gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
+ - gcovr -r ../ --exclude-throw-branches
+ artifacts:
+ paths:
+ - coverage
+ expire_in: 8d
+ only:
+ - branches
+ - tags
+ tags:
Doc:
- stage: DocCoverage
- script:
- - export LD_LIBRARY_PATH=/usr/lib
- - export GIT_SSL_NO_VERIFY=1
- - env
- - git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
- - cd PhoenixTex2Html
- - ./install.sh /usr
- - cd ..
- - rm -fr PhoenixTex2Html
- - rm -fr build public
- - mkdir -p build
- - mkdir -p public/html
- - cd build
- - >
- cmake ..
- -DCMAKE_INSTALL_PREFIX=/usr
- -DDOC_MODE=yes
- - make all
- - make doc
- - scp -r doc/platex/* ../public
- - scp -r doc/html/* ../public/html
- #- scp -r doc/html/* ../public
- artifacts:
- paths:
- - public
- expire_in: 8d
- only:
- - branches
- - tags
- tags:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+ stage: DocCoverage
+ script:
+ - export LD_LIBRARY_PATH=/usr/lib
+ - export GIT_SSL_NO_VERIFY=1
+ - env
+ - git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
+ - cd PhoenixTex2Html
+ - ./install.sh /usr
+ - cd ..
+ - rm -fr PhoenixTex2Html
+ - rm -fr build public
+ - mkdir -p build
+ - mkdir -p public/html
+ - cd build
+ - >
+ cmake ..
+ -DCMAKE_INSTALL_PREFIX=/usr
+ -DDOC_MODE=yes
+ - make all
+ - make doc
+ - scp -r doc/platex/* ../public
+ - scp -r doc/html/* ../public/html
+ #- scp -r doc/html/* ../public
+ artifacts:
+ paths:
+ - public
+ expire_in: 8d
+ only:
+ - branches
+ - tags
+ tags:
pages:
- stage: deploy
- script:
- - mv coverage public/
- artifacts:
- paths:
- - public
- expire_in: 8d
+ stage: deploy
+ script:
+ - mv coverage public/
+ artifacts:
+ paths:
+ - public
+ expire_in: 8d
+
+Ubuntu 2004:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2004_gcc9:0.1
+ stage: DocCoverage
+ script:
+ - ./cmake/phoenix_createPackages.sh DEB package_ubuntu2004 false
+ artifacts:
+ paths:
+ - package_ubuntu2004
+ expire_in: 8d
+ only:
+ #- branches
+ - tags
+Ubuntu 2204:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_gcc11:0.1
+ stage: DocCoverage
+ script:
+ - ./cmake/phoenix_createPackages.sh DEB package_ubuntu2204 false
+ artifacts:
+ paths:
+ - package_ubuntu2204
+ expire_in: 8d
+ only:
+ #- branches
+ - tags
+
+Fedora 31:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora31:0.1
+ stage: DocCoverage
+ script:
+ - ./cmake/phoenix_createPackages.sh RPM package_fedora31 false
+ artifacts:
+ paths:
+ - package_fedora31
+ expire_in: 8d
+ only:
+ #- branches
+ - tags
+Fedora 36:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora36:0.1
+ stage: DocCoverage
+ script:
+ - ./cmake/phoenix_createPackages.sh RPM package_fedora36 false
+ artifacts:
+ paths:
+ - package_fedora36
+ expire_in: 8d
+ only:
+ #- branches
+ - tags
+
+Release:
+ image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_python3:0.1
+ stage: deploy
+ script:
+ - echo "Let's do some release"
+ - uname -a
+ - python3 ./cmake/createReleaseCurl.py -n ${CI_PROJECT_TITLE} -i ${CI_PROJECT_ID} -t ${CI_COMMIT_TAG} -p ${CI_RELEASE_TOKEN}
+ only:
+ #- branches
+ - tags
diff --git a/cmake/createReleaseCurl.py b/cmake/createReleaseCurl.py
new file mode 100644
index 0000000..657a4fe
--- /dev/null
+++ b/cmake/createReleaseCurl.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import sys
+import os
+import subprocess
+import json
+import glob
+import argparse
+
+server="gitlab.in2p3.fr"
+apiProject="/api/v4/projects/"
+
+
+def getListArchiveFile(inputDirectory, projectName):
+ '''
+ Get the list of the archives file for a given project name
+ Parameters:
+ inputDirectory : directory (for one OS) where all the binary packages are created
+ projectName : name of the project
+ Return:
+ list of corresponding binary packages
+ '''
+ listPackage = glob.glob(inputDirectory + "/*/" + projectName + "-*" )
+ print(listPackage)
+ return listPackage
+
+
+#curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads"
+def uploadFileCommand(projectIdOwn, fileName, token):
+ '''
+ Upload a file in the given gitlab project
+ Parameters:
+ projectIdOwn : id of the project
+ fileName : name of the file to be uploaded
+ token : token to be used to create the release
+ Return:
+ url of the uploaded file
+ '''
+ print("Upload file",fileName)
+ URL = '"https://'+server+apiProject+str(projectIdOwn)+'/uploads"'
+ privateToken='--header "PRIVATE-TOKEN: '+token+'" '
+ fileCmd = '--form "file=@'+fileName+'" '
+ command = "curl --insecure --request POST " + privateToken + fileCmd + URL
+ #print("Updload file command :", command)
+ output = subprocess.getoutput(command)
+ print("Upload file output :",output)
+ outputDico = json.loads('{' + output.split("{")[1])
+ outputFile = outputDico["full_path"]
+ outputUrl = outputDico["full_path"]
+ print("Upload file output file ",outputFile, ", url :",outputUrl)
+ return outputFile, outputUrl
+
+
+
+#{ "name": "hoge", "url": "https://google.com", "filepath": "/binaries/linux-amd64", "link_type":"other" }
+
+def getListArchiveLinks(projectIdOwn, listPackage, useComma, token):
+ '''
+ Create the list of link to the archive uploaded binary packages
+ Parameters:
+ projectIdOwn : id of the project to be used
+ listPackage : list of the binary packages to be uploaded and linked
+ useComma : True to add a comma in the first place, false otherwise
+ token : token to be used to create the release
+ Return:
+ string of the list of archive links corresponding to the given list of packages
+ '''
+ linksData = ""
+ for packageName in listPackage:
+ if useComma:
+ linksData += ", "
+ baseName = os.path.basename(packageName)
+ packageOsName = packageName.split('/')[-2]
+ nameData = "\"name\": \"" + packageOsName + " " + baseName+"\""
+
+ uploadedFullPath, outputUrl = uploadFileCommand(projectIdOwn, packageName, token)
+ print("outputUrl =",outputUrl)
+ fullPathToFile = 'https://'+server+"/"+uploadedFullPath
+ #filePathData = '"filepath": "'+fullPathToFile+'"'
+ linkType = '"link_type": "other" '
+ urlData = '"url": "'+fullPathToFile+'"'
+
+ addDataLink = '{ '+nameData + ", " + urlData + ", " + linkType +'}'
+ print("addDataLink =",addDataLink)
+ linksData += addDataLink
+ useComma = True
+
+ return linksData
+
+
+def createReleaseCurlCommand(projectIdOwn, projectName, projectTagName, basePackageDir, token):
+ '''
+ Create a release of the given project
+ Parameters:
+ projectIdOwn : id of the project on gitlab
+ projectName : name of the project
+ projectTagName : tag name of the project
+ basePackageDir : base directory wher eto find binary packages
+ token : token to be used to create the release
+ '''
+ postURL="--request POST https://"+server+apiProject+str(projectIdOwn)+"/releases"
+ header="--header 'Content-Type: application/json' "
+ privateToken='--header "PRIVATE-TOKEN: '+token+'" '
+ versionName="version " + projectTagName
+ dataStr = "--data '{ \"name\": \"Release "+versionName+"\"," + "\"tag_name\": \""+projectTagName+"\","
+ dataStr += "\"description\": \"Automatic release, "+versionName+"\""
+
+ useComma = False
+ linksData = ", \"assets\": { \"links\": ["
+ listArchiveFile = getListArchiveFile(basePackageDir + "/", projectName)
+ linksData += getListArchiveLinks(projectIdOwn, listArchiveFile, useComma, token)
+ linksData += "] } "
+ dataStr += linksData
+ dataStr += "}' "
+ command = "curl --insecure " + header + privateToken + dataStr + postURL
+ print("Create Release command :",command)
+ output = subprocess.getoutput(command)
+ print("Release Output :",output)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Program which creates release with given tag")
+ parser.add_argument('-n', '--projectname', help="Name of the current projet", required=True, type=str)
+ parser.add_argument('-i', '--projectid', help="Id of the current projet", required=True, type=int)
+ parser.add_argument('-t', '--tag', help="Tag to be used to create the current release", required=True, type=str)
+ parser.add_argument('-p', '--token', help="Token to be used to create the release", required=True, type=str)
+ parser.add_argument('-d', '--packagedirectory', help="Directory where to find packages", required=False, type=str, default="./")
+ args = parser.parse_args()
+
+ createReleaseCurlCommand(args.projectid, args.projectname, args.tag, args.packagedirectory, args.token)
+
+
+
+
diff --git a/cmake/phoenix_createPackages.sh b/cmake/phoenix_createPackages.sh
new file mode 100755
index 0000000..6fa3a1a
--- /dev/null
+++ b/cmake/phoenix_createPackages.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+PACKAGE_TYPE="$1"
+OUTPUT_LINUX_DIR="$2"
+USE_VECTORISATION="$3"
+
+DIR_SOFT_NAME=`basename ${PWD}`
+
+rm -fr ${OUTPUT_LINUX_DIR}
+mkdir -p ${OUTPUT_LINUX_DIR}
+
+function createBinaryArchive(){
+ EXTENTION="$1"
+ echo "Create binary archive for extension '${EXTENTION}'"
+
+ if [ ! -z "$1" ]
+ then
+ cd ..
+ mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
+ cd $DIR_SOFT_NAME-${EXTENTION}
+ fi
+
+ if [ -d build ]
+ then
+ echo "Remove existing directory build"
+ rm -fr build
+ fi
+
+ mkdir -p build
+ cd build
+
+ if [ -z "$1" ]
+ then
+ cmake .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+ else
+ cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+ fi
+# make -j `nproc`
+ make
+ cpack
+ cd ..
+ if [ ! -z "$1" ]
+ then
+ cd ..
+ mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
+ cd $DIR_SOFT_NAME
+ fi
+}
+
+function createPackage(){
+ PACKAGE_TYPE="$1"
+ EXTENTION="$2"
+ echo "Create Package '${PACKAGE_TYPE}' for extension '${EXTENTION}'"
+
+ if [ ! -z "$1" ]
+ then
+ cd ..
+ mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
+ cd $DIR_SOFT_NAME-${EXTENTION}
+ fi
+
+ if [ -d build ]
+ then
+ echo "Remove existing directory build"
+ rm -fr build
+ fi
+ mkdir -p build
+ cd build
+# The -DCMAKE_INSTALL_PREFIX=/usr is needed because the install prefix of CPack is /usr
+ if [ -z "$2" ]
+ then
+ cmake .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+ else
+ cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+ fi
+# make -j `nproc`
+ make
+ cpack
+ cd ..
+ if [ ! -z "$1" ]
+ then
+ cd ..
+ mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
+ cd $DIR_SOFT_NAME
+ fi
+}
+
+echo "Create package, activate extension '${USE_VECTORISATION}'"
+if [[ "$USE_VECTORISATION" == "true"* ]]
+then
+ echo "Create packages for set of extension (AVX, AVX2, AVX512F, ...)"
+ for extension in "AVX512_2" "AVX512VPOPCNTDQ" "AVX512BW" "AVX512PF" "AVX512ER" "AVX512F" "AVX2" "AVX" "SSE4.2" "SSE4.1" "SSSE3" "SSE2" "NOVECTORIZATION"
+ do
+ createPackage ${PACKAGE_TYPE} ${extension}
+ cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+ createBinaryArchive ${extension}
+ cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+ done
+else
+ echo "Create packages for generic extension"
+ createPackage ${PACKAGE_TYPE}
+ cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+ createBinaryArchive
+ cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+fi
+
+
+
+
+
--
GitLab