Add scripts to make automatic release within CI

.gitlab-ci.yml

-image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
 
 stages:
 - BuildTestinstall
@@ -6,103 +5,166 @@ stages:
 - deploy
 
 dailyBuildMasterAll:
-    #cache:
-        #untracked: true
-        #paths:
-        #- build/
-
-    stage: BuildTestinstall
-    script:
-    - export LD_LIBRARY_PATH=/usr/lib
-    - env
-    - mkdir -p build
-    - cd build
-    - >
-        cmake ..
-        -DCMAKE_INSTALL_PREFIX=/usr
-        -DSELF_TESTS_MODE=yes
-    - make all
-    - make install
-    - make test
-    only:
-    - branches
-    - tags
-    tags:
+        #cache:
+                #untracked: true
+                #paths:
+                #- build/
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+        stage: BuildTestinstall
+        script:
+        - export LD_LIBRARY_PATH=/usr/lib
+        - env
+        - mkdir -p build
+        - cd build
+        - >
+                cmake ..
+                -DCMAKE_INSTALL_PREFIX=/usr
+                -DSELF_TESTS_MODE=yes
+        - make all
+        - make install
+        - make test
+        only:
+        - branches
+        - tags
+        tags:
 
 TestCoverage:
-    stage: DocCoverage
-    script:
-    - export LD_LIBRARY_PATH=/usr/lib
-    - env
-    - mkdir -p build coverage
-    - cd build
-    - >
-        cmake ..
-        -DCMAKE_INSTALL_PREFIX=/usr
-        -DSELF_TESTS_MODE=yes
-        -DCMAKE_BUILD_TYPE=Coverage
-    - make all
-    - make install
-    - make test
-    - make run_all
-    - make ExperimentalCoverage
-    - cd ../coverage
-    - gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
-    - gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
-    - gcovr -r ../ --exclude-throw-branches
-    artifacts:
-        paths:
-        - coverage
-        expire_in: 8d
-    only:
-    - branches
-    - tags
-    tags:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+        stage: DocCoverage
+        script:
+        - export LD_LIBRARY_PATH=/usr/lib
+        - env
+        - mkdir -p build coverage
+        - cd build
+        - >
+                cmake ..
+                -DCMAKE_INSTALL_PREFIX=/usr
+                -DSELF_TESTS_MODE=yes
+                -DCMAKE_BUILD_TYPE=Coverage
+        - make all
+        - make install
+        - make test
+        - make run_all
+        - make ExperimentalCoverage
+        - cd ../coverage
+        - gcovr -r ../ --exclude-throw-branches --html cov_report.html --html-details
+        - gcovr -r ../ --exclude-throw-branches --xml Coverage.xml
+        - gcovr -r ../ --exclude-throw-branches
+        artifacts:
+                paths:
+                - coverage
+                expire_in: 8d
+        only:
+        - branches
+        - tags
+        tags:
 
 Doc:
-    stage: DocCoverage
-    script:
-    - export LD_LIBRARY_PATH=/usr/lib
-    - export GIT_SSL_NO_VERIFY=1
-    - env
-    - git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
-    - cd PhoenixTex2Html
-    - ./install.sh /usr
-    - cd ..
-    - rm -fr PhoenixTex2Html
-    - rm -fr build public
-    - mkdir -p build
-    - mkdir -p public/html
-    - cd build
-    - >
-        cmake ..
-        -DCMAKE_INSTALL_PREFIX=/usr
-        -DDOC_MODE=yes
-    - make all
-    - make doc
-    - scp -r doc/platex/* ../public
-    - scp -r doc/html/* ../public/html
-    #- scp -r doc/html/* ../public
-    artifacts:
-      paths:
-      - public
-      expire_in: 8d
-    only:
-    - branches
-    - tags
-    tags:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenix/phoenix_minimal:0.2
+        stage: DocCoverage
+        script:
+        - export LD_LIBRARY_PATH=/usr/lib
+        - export GIT_SSL_NO_VERIFY=1
+        - env
+        - git clone https://gitlab.in2p3.fr/CTA-LAPP/PHOENIX_LIBS/PhoenixTex2Html.git
+        - cd PhoenixTex2Html
+        - ./install.sh /usr
+        - cd ..
+        - rm -fr PhoenixTex2Html
+        - rm -fr build public
+        - mkdir -p build
+        - mkdir -p public/html
+        - cd build
+        - >
+                cmake ..
+                -DCMAKE_INSTALL_PREFIX=/usr
+                -DDOC_MODE=yes
+        - make all
+        - make doc
+        - scp -r doc/platex/* ../public
+        - scp -r doc/html/* ../public/html
+        #- scp -r doc/html/* ../public
+        artifacts:
+                paths:
+                - public
+                expire_in: 8d
+        only:
+        - branches
+        - tags
+        tags:
 
 
 pages:
-  stage: deploy
-  script:
-    - mv coverage public/
-  artifacts:
-    paths:
-      - public
-    expire_in: 8d
+        stage: deploy
+        script:
+        - mv coverage public/
+        artifacts:
+                paths:
+                - public
+                expire_in: 8d
+
 
+Ubuntu 2004:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2004_gcc9:0.1
+        stage: DocCoverage
+        script:
+                - ./cmake/phoenix_createPackages.sh DEB package_ubuntu2004 false
+        artifacts:
+                paths:
+                - package_ubuntu2004
+                expire_in: 8d
+        only:
+        #- branches
+        - tags
 
+Ubuntu 2204:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_gcc11:0.1
+        stage: DocCoverage
+        script:
+                - ./cmake/phoenix_createPackages.sh DEB package_ubuntu2204 false
+        artifacts:
+                paths:
+                - package_ubuntu2204
+                expire_in: 8d
+        only:
+        #- branches
+        - tags
+        
+Fedora 31:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora31:0.1
+        stage: DocCoverage
+        script:
+                - ./cmake/phoenix_createPackages.sh RPM package_fedora31 false
+        artifacts:
+                paths:
+                - package_fedora31
+                expire_in: 8d
+        only:
+        #- branches
+        - tags
 
+Fedora 36:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_fedora36:0.1
+        stage: DocCoverage
+        script:
+                - ./cmake/phoenix_createPackages.sh RPM package_fedora36 false
+        artifacts:
+                paths:
+                - package_fedora36
+                expire_in: 8d
+        only:
+        #- branches
+        - tags
+        
+Release:
+        image: gitlab-registry.in2p3.fr/cta-lapp/phoenix_libs/phoenixdocker/package_ubuntu2204_python3:0.1
+        stage: deploy
+        script:
+        - echo "Let's do some release"
+        - uname -a
+        - python3 ./cmake/createReleaseCurl.py -n ${CI_PROJECT_TITLE} -i ${CI_PROJECT_ID} -t ${CI_COMMIT_TAG} -p ${CI_RELEASE_TOKEN}
+        only:
+        #- branches
+        - tags
 
 

cmake/createReleaseCurl.py

+#!/usr/bin/env python
+# coding: utf-8
+
+import sys
+import os
+import subprocess
+import json
+import glob
+import argparse
+
+server="gitlab.in2p3.fr"
+apiProject="/api/v4/projects/"
+
+
+def getListArchiveFile(inputDirectory, projectName):
+	'''
+	Get the list of the archives file for a given project name
+	Parameters:
+		inputDirectory : directory (for one OS) where all the binary packages are created
+		projectName : name of the project
+	Return:
+		list of corresponding binary packages
+	'''
+	listPackage = glob.glob(inputDirectory + "/*/" + projectName + "-*" )
+	print(listPackage)
+	return listPackage
+
+
+#curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads"
+def uploadFileCommand(projectIdOwn, fileName, token):
+	'''
+	Upload a file in the given gitlab project
+	Parameters:
+		projectIdOwn : id of the project
+		fileName : name of the file to be uploaded
+		token : token to be used to create the release
+	Return:
+		url of the uploaded file
+	'''
+	print("Upload file",fileName)
+	URL = '"https://'+server+apiProject+str(projectIdOwn)+'/uploads"'
+	privateToken='--header "PRIVATE-TOKEN: '+token+'" '
+	fileCmd = '--form "file=@'+fileName+'" '
+	command = "curl --insecure --request POST " + privateToken + fileCmd + URL
+	#print("Updload file command :", command)
+	output = subprocess.getoutput(command)
+	print("Upload file output :",output)
+	outputDico = json.loads('{' + output.split("{")[1])
+	outputFile = outputDico["full_path"]
+	outputUrl = outputDico["full_path"]
+	print("Upload file output file ",outputFile, ", url :",outputUrl)
+	return outputFile, outputUrl
+
+
+
+#{ "name": "hoge", "url": "https://google.com", "filepath": "/binaries/linux-amd64", "link_type":"other" }
+
+def getListArchiveLinks(projectIdOwn, listPackage, useComma, token):
+	'''
+	Create the list of link to the archive uploaded binary packages
+	Parameters:
+		projectIdOwn : id of the project to be used
+		listPackage : list of the binary packages to be uploaded and linked
+		useComma : True to add a comma in the first place, false otherwise
+		token : token to be used to create the release
+	Return:
+		string of the list of archive links corresponding to the given list of packages
+	'''
+	linksData = ""
+	for packageName in listPackage:
+		if useComma:
+			linksData += ", "
+		baseName = os.path.basename(packageName)
+		packageOsName = packageName.split('/')[-2]
+		nameData = "\"name\": \"" + packageOsName + " " + baseName+"\""
+		
+		uploadedFullPath, outputUrl = uploadFileCommand(projectIdOwn, packageName, token)
+		print("outputUrl =",outputUrl)
+		fullPathToFile = 'https://'+server+"/"+uploadedFullPath
+		#filePathData = '"filepath": "'+fullPathToFile+'"' 
+		linkType = '"link_type": "other" '
+		urlData = '"url": "'+fullPathToFile+'"'
+		
+		addDataLink = '{ '+nameData + ", " + urlData + ", " + linkType +'}'
+		print("addDataLink =",addDataLink)
+		linksData += addDataLink
+		useComma = True
+	
+	return linksData
+
+
+def createReleaseCurlCommand(projectIdOwn, projectName, projectTagName, basePackageDir, token):
+	'''
+	Create a release of the given project
+	Parameters:
+		projectIdOwn : id of the project on gitlab
+		projectName : name of the project
+		projectTagName : tag name of the project
+		basePackageDir : base directory wher eto find binary packages
+		token : token to be used to create the release
+	'''
+	postURL="--request POST https://"+server+apiProject+str(projectIdOwn)+"/releases"
+	header="--header 'Content-Type: application/json' "
+	privateToken='--header "PRIVATE-TOKEN: '+token+'" '
+	versionName="version " + projectTagName
+	dataStr = "--data '{ \"name\": \"Release "+versionName+"\"," + "\"tag_name\": \""+projectTagName+"\","
+	dataStr += "\"description\": \"Automatic release, "+versionName+"\""
+	
+	useComma = False
+	linksData = ", \"assets\": { \"links\": ["
+	listArchiveFile = getListArchiveFile(basePackageDir + "/", projectName)
+	linksData += getListArchiveLinks(projectIdOwn, listArchiveFile, useComma, token)
+	linksData += "] } "
+	dataStr += linksData
+	dataStr += "}' "
+	command = "curl --insecure " + header + privateToken + dataStr + postURL
+	print("Create Release command :",command)
+	output = subprocess.getoutput(command)
+	print("Release Output :",output)
+
+
+if __name__ == "__main__":
+	parser = argparse.ArgumentParser(description="Program which creates release with given tag")
+	parser.add_argument('-n', '--projectname', help="Name of the current projet", required=True, type=str)
+	parser.add_argument('-i', '--projectid', help="Id of the current projet", required=True, type=int)
+	parser.add_argument('-t', '--tag', help="Tag to be used to create the current release", required=True, type=str)
+	parser.add_argument('-p', '--token', help="Token to be used to create the release", required=True, type=str)
+	parser.add_argument('-d', '--packagedirectory', help="Directory where to find packages", required=False, type=str, default="./")
+	args = parser.parse_args()
+	
+	createReleaseCurlCommand(args.projectid, args.projectname, args.tag, args.packagedirectory, args.token)
+
+
+
+

cmake/phoenix_createPackages.sh

+#!/bin/bash
+
+PACKAGE_TYPE="$1"
+OUTPUT_LINUX_DIR="$2"
+USE_VECTORISATION="$3"
+
+DIR_SOFT_NAME=`basename ${PWD}`
+
+rm -fr ${OUTPUT_LINUX_DIR}
+mkdir -p ${OUTPUT_LINUX_DIR}
+
+function createBinaryArchive(){
+	EXTENTION="$1"
+	echo "Create binary archive for extension '${EXTENTION}'"
+	
+	if [ ! -z "$1" ]
+	then
+		cd ..
+		mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
+		cd $DIR_SOFT_NAME-${EXTENTION}
+	fi
+	
+	if [ -d build ]
+	then
+		echo "Remove existing directory build"
+		rm -fr build
+	fi
+	
+	mkdir -p build
+	cd build
+	
+	if [ -z "$1" ]
+	then
+		cmake .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+	else
+		cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+	fi
+# 	make -j `nproc`
+	make
+	cpack
+	cd ..
+	if [ ! -z "$1" ]
+	then
+		cd ..
+		mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
+		cd $DIR_SOFT_NAME
+	fi
+}
+
+function createPackage(){
+	PACKAGE_TYPE="$1"
+	EXTENTION="$2"
+	echo "Create Package '${PACKAGE_TYPE}' for extension '${EXTENTION}'"
+	
+	if [ ! -z "$1" ]
+	then
+		cd ..
+		mv $DIR_SOFT_NAME $DIR_SOFT_NAME-${EXTENTION}
+		cd $DIR_SOFT_NAME-${EXTENTION}
+	fi
+	
+	if [ -d build ]
+	then
+		echo "Remove existing directory build"
+		rm -fr build
+	fi
+	mkdir -p build
+	cd build
+# 	The  -DCMAKE_INSTALL_PREFIX=/usr is needed because the install prefix of CPack is /usr
+	if [ -z "$2" ]
+	then
+		cmake .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+	else
+		cmake .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr || cmake3 .. -D${EXTENTION}=yes -DPROGRAM_EXTENSION=${EXTENTION} -DCREATE_${PACKAGE_TYPE}=yes -DSELF_TESTS_MODE=no -DCMAKE_INSTALL_PREFIX=/usr
+	fi
+# 	make -j `nproc`
+	make
+	cpack
+	cd ..
+	if [ ! -z "$1" ]
+	then
+		cd ..
+		mv $DIR_SOFT_NAME-${EXTENTION} $DIR_SOFT_NAME
+		cd $DIR_SOFT_NAME
+	fi
+}
+
+echo "Create package, activate extension '${USE_VECTORISATION}'"
+if [[ "$USE_VECTORISATION" == "true"* ]]
+then
+	echo "Create packages for set of extension (AVX, AVX2, AVX512F, ...)"
+	for extension in "AVX512_2" "AVX512VPOPCNTDQ" "AVX512BW" "AVX512PF" "AVX512ER" "AVX512F" "AVX2" "AVX" "SSE4.2" "SSE4.1" "SSSE3" "SSE2" "NOVECTORIZATION"
+	do
+		createPackage ${PACKAGE_TYPE} ${extension}
+		cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+		createBinaryArchive ${extension}
+		cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+	done
+else
+	echo "Create packages for generic extension"
+	createPackage ${PACKAGE_TYPE}
+	cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+	createBinaryArchive
+	cp build/${DIR_SOFT_NAME}-* "${OUTPUT_LINUX_DIR}"
+fi
+
+
+
+
+