Commit dbe04e72 authored by Pierre Aubert's avatar Pierre Aubert
Browse files

Add scripts to make automatic release within CI

parent 15e701ac
Pipeline #193894 passed with stages
in 12 minutes and 59 seconds
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
stages:
- BuildTestinstall
......@@ -6,103 +5,166 @@ stages:
- deploy
dailyBuildMasterAll:
#cache:
#untracked: true
#paths:
#- build/
stage: BuildTestinstall
script:
- export LD_LIBRARY_PATH=/usr/lib
- env
- mkdir -p build
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DSELF_TESTS_MODE=yes
- make all
- make install
- make test
only:
- branches
- tags
tags:
#cache:
#untracked: true
#paths:
#- build/
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
stage: BuildTestinstall
script:
- export LD_LIBRARY_PATH=/usr/lib
- env
- mkdir -p build
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DSELF_TESTS_MODE=yes
- make all
- make install
- make test
only:
- branches
- tags
tags:
TestCoverage:
stage: DocCoverage
script:
- export LD_LIBRARY_PATH=/usr/lib
- env
- mkdir -p build coverage
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DSELF_TESTS_MODE=yes
-DCMAKE_BUILD_TYPE=Coverage
- make all
- make install
- make test
- make run_all
- make ExperimentalCoverage
- cd ../coverage
- gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
- gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
- gcovr -r ../ --exclude-throw-branches
artifacts:
paths:
- coverage
expire_in: 8d
only:
- branches
- tags
tags:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
stage: DocCoverage
script:
- export LD_LIBRARY_PATH=/usr/lib
- env
- mkdir -p build coverage
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DSELF_TESTS_MODE=yes
-DCMAKE_BUILD_TYPE=Coverage
- make all
- make install
- make test
- make run_all
- make ExperimentalCoverage
- cd ../coverage
- gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
- gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
- gcovr -r ../ --exclude-throw-branches
artifacts:
paths:
- coverage
expire_in: 8d
only:
- branches
- tags
tags:
Doc:
stage: DocCoverage
script:
- export LD_LIBRARY_PATH=/usr/lib
- export GIT_SSL_NO_VERIFY=1
- env
- git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
- cd PhoenixTex2Html
- ./install.sh /usr
- cd ..
- rm -fr PhoenixTex2Html
- rm -fr build public
- mkdir -p build
- mkdir -p public/html
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DDOC_MODE=yes
- make all
- make doc
- scp -r doc/platex/* ../public
- scp -r doc/html/* ../public/html
#- scp -r doc/html/* ../public
artifacts:
paths:
- public
expire_in: 8d
only:
- branches
- tags
tags:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
stage: DocCoverage
script:
- export LD_LIBRARY_PATH=/usr/lib
- export GIT_SSL_NO_VERIFY=1
- env
- git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
- cd PhoenixTex2Html
- ./install.sh /usr
- cd ..
- rm -fr PhoenixTex2Html
- rm -fr build public
- mkdir -p build
- mkdir -p public/html
- cd build
- >
cmake ..
-DCMAKE_INSTALL_PREFIX=/usr
-DDOC_MODE=yes
- make all
- make doc
- scp -r doc/platex/* ../public
- scp -r doc/html/* ../public/html
#- scp -r doc/html/* ../public
artifacts:
paths:
- public
expire_in: 8d
only:
- branches
- tags
tags:
pages:
stage: deploy
script:
- mv coverage public/
artifacts:
paths:
- public
expire_in: 8d
stage: deploy
script:
- mv coverage public/
artifacts:
paths:
- public
expire_in: 8d
Ubuntu 2004:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2004_gcc9:0.1
stage: DocCoverage
script:
- ./cmake/phoenix_createPackages.sh DEB package_ubuntu2004 false
artifacts:
paths:
- package_ubuntu2004
expire_in: 8d
only:
#- branches
- tags
Ubuntu 2204:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_gcc11:0.1
stage: DocCoverage
script:
- ./cmake/phoenix_createPackages.sh DEB package_ubuntu2204 false
artifacts:
paths:
- package_ubuntu2204
expire_in: 8d
only:
#- branches
- tags
Fedora 31:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora31:0.1
stage: DocCoverage
script:
- ./cmake/phoenix_createPackages.sh RPM package_fedora31 false
artifacts:
paths:
- package_fedora31
expire_in: 8d
only:
#- branches
- tags
Fedora 36:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora36:0.1
stage: DocCoverage
script:
- ./cmake/phoenix_createPackages.sh RPM package_fedora36 false
artifacts:
paths:
- package_fedora36
expire_in: 8d
only:
#- branches
- tags
Release:
image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_python3:0.1
stage: deploy
script:
- echo "Let's do some release"
- uname -a
- python3 ./cmake/createReleaseCurl.py -n ${CI_PROJECT_TITLE} -i ${CI_PROJECT_ID} -t ${CI_COMMIT_TAG} -p ${CI_RELEASE_TOKEN}
only:
#- branches
- tags
#!/usr/bin/env python
# coding: utf-8
import sys
import os
import subprocess
import json
import glob
import argparse
server="gitlab.in2p3.fr"
apiProject="/api/v4/projects/"
def getListArchiveFile(inputDirectory, projectName):
'''
Get the list of the archives file for a given project name
Parameters:
inputDirectory : directory (for one OS) where all the binary packages are created
projectName : name of the project
Return:
list of corresponding binary packages
'''
listPackage = glob.glob(inputDirectory + "/*/" + projectName + "-*" )
print(listPackage)
return listPackage
#curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads"
def uploadFileCommand(projectIdOwn, fileName, token):
'''
Upload a file in the given gitlab project
Parameters:
projectIdOwn : id of the project
fileName : name of the file to be uploaded
token : token to be used to create the release
Return:
url of the uploaded file
'''
print("Upload file",fileName)
URL = '"https://'+server+apiProject+str(projectIdOwn)+'/uploads"'
privateToken='--header "PRIVATE-TOKEN: '+token+'" '
fileCmd = '--form "file=@'+fileName+'" '
command = "curl --insecure --request POST " + privateToken + fileCmd + URL
#print("Updload file command :", command)
output = subprocess.getoutput(command)
print("Upload file output :",output)
outputDico = json.loads('{' + output.split("{")[1])
outputFile = outputDico["full_path"]
outputUrl = outputDico["full_path"]
print("Upload file output file ",outputFile, ", url :",outputUrl)
return outputFile, outputUrl
#{ "name": "hoge", "url": "https://google.com", "filepath": "/binaries/linux-amd64", "link_type":"other" }
def getListArchiveLinks(projectIdOwn, listPackage, useComma, token):
'''
Create the list of link to the archive uploaded binary packages
Parameters:
projectIdOwn : id of the project to be used
listPackage : list of the binary packages to be uploaded and linked
useComma : True to add a comma in the first place, false otherwise
token : token to be used to create the release
Return:
string of the list of archive links corresponding to the given list of packages
'''
linksData = ""
for packageName in listPackage:
if useComma:
linksData += ", "
baseName = os.path.basename(packageName)
packageOsName = packageName.split('/')[-2]
nameData = "\"name\": \"" + packageOsName + " " + baseName+"\""
uploadedFullPath, outputUrl = uploadFileCommand(projectIdOwn, packageName, token)
print("outputUrl =",outputUrl)
fullPathToFile = 'https://'+server+"/"+uploadedFullPath
#filePathData = '"filepath": "'+fullPathToFile+'"'
linkType = '"link_type": "other" '
urlData = '"url": "'+fullPathToFile+'"'
addDataLink = '{ '+nameData + ", " + urlData + ", " + linkType +'}'
print("addDataLink =",addDataLink)
linksData += addDataLink
useComma = True
return linksData
def createReleaseCurlCommand(projectIdOwn, projectName, projectTagName, basePackageDir, token):
'''
Create a release of the given project
Parameters:
projectIdOwn : id of the project on gitlab
projectName : name of the project
projectTagName : tag name of the project
basePackageDir : base directory wher eto find binary packages
token : token to be used to create the release
'''
postURL="--request POST https://"+server+apiProject+str(projectIdOwn)+"/releases"
header="--header 'Content-Type: application/json' "
privateToken='--header "PRIVATE-TOKEN: '+token+'" '
versionName="version " + projectTagName
dataStr = "--data '{ \"name\": \"Release "+versionName+"\"," + "\"tag_name\": \""+projectTagName+"\","
dataStr += "\"description\": \"Automatic release, "+versionName+"\""
useComma = False
linksData = ", \"assets\": { \"links\": ["
listArchiveFile = getListArchiveFile(basePackageDir + "/", projectName)
linksData += getListArchiveLinks(projectIdOwn, listArchiveFile, useComma, token)
linksData += "] } "
dataStr += linksData
dataStr += "}' "
command = "curl --insecure " + header + privateToken + dataStr + postURL
print("Create Release command :",command)
output = subprocess.getoutput(command)
print("Release Output :",output)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Program which creates release with given tag")
parser.add_argument('-n', '--projectname', help="Name of the current projet", required=True, type=str)
parser.add_argument('-i', '--projectid', help="Id of the current projet", required=True, type=int)
parser.add_argument('-t', '--tag', help="Tag to be used to create the current release", required=True, type=str)
parser.add_argument('-p', '--token', help="Token to be used to create the release", required=True, type=str)
parser.add_argument('-d', '--packagedirectory', help="Directory where to find packages", required=False, type=str, default="./")
args = parser.parse_args()
createReleaseCurlCommand(args.projectid, args.projectname, args.tag, args.packagedirectory, args.token)
#!/bin/bash
PACKAGE_TYPE="$1"
OUTPUT_LINUX_DIR="$2"
USE_VECTORISATION="$3"
DIR_SOFT_NAME=`basename ${PWD}`
rm -fr ${OUTPUT_LINUX_DIR}
mkdir -p ${OUTPUT_LINUX_DIR}
function createBinaryArchive(){
EXTENTION="$1"
echo "Create binary archive for extension '${EXTENTION}'"
if [ ! -z "$1" ]
then
cd ..
mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
cd $DIR_SOFT_NAME-${EXTENTION}
fi
if [ -d build ]
then
echo "Remove existing directory build"
rm -fr build
fi
mkdir -p build
cd build
if [ -z "$1" ]
then
cmake .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
else
cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
fi
# make -j `nproc`
make
cpack
cd ..
if [ ! -z "$1" ]
then
cd ..
mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
cd $DIR_SOFT_NAME
fi
}
function createPackage(){
PACKAGE_TYPE="$1"
EXTENTION="$2"
echo "Create Package '${PACKAGE_TYPE}' for extension '${EXTENTION}'"
if [ ! -z "$1" ]
then
cd ..
mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
cd $DIR_SOFT_NAME-${EXTENTION}
fi
if [ -d build ]
then
echo "Remove existing directory build"
rm -fr build
fi
mkdir -p build
cd build
# The -DCMAKE_INSTALL_PREFIX=/usr is needed because the install prefix of CPack is /usr
if [ -z "$2" ]
then
cmake .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
else
cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
fi
# make -j `nproc`
make
cpack
cd ..
if [ ! -z "$1" ]
then
cd ..
mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
cd $DIR_SOFT_NAME
fi
}
echo "Create package, activate extension '${USE_VECTORISATION}'"
if [[ "$USE_VECTORISATION" == "true"* ]]
then
echo "Create packages for set of extension (AVX, AVX2, AVX512F, ...)"
for extension in "AVX512_2" "AVX512VPOPCNTDQ" "AVX512BW" "AVX512PF" "AVX512ER" "AVX512F" "AVX2" "AVX" "SSE4.2" "SSE4.1" "SSSE3" "SSE2" "NOVECTORIZATION"
do
createPackage ${PACKAGE_TYPE} ${extension}
cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
createBinaryArchive ${extension}
cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
done
else
echo "Create packages for generic extension"
createPackage ${PACKAGE_TYPE}
cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
createBinaryArchive
cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment