.gitlab-ci.yml 11.2 KB
Newer Older
1
# === Global CI settings ===
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
2

3
variables:
4
  BUILD_ENV: "${CI_REGISTRY_IMAGE}:testing-${CI_PIPELINE_IID}-build-environment"
5 6 7 8 9 10 11
  CI_SOPHYA_ENV: "${CI_REGISTRY_IMAGE}:testing-${CI_PIPELINE_IID}"
  DEBUG_SOPHYA_ENV: "${CI_REGISTRY_IMAGE}:testing"
  LOCAL_SOPHYABASE: "sophyabase"
  OFFICIAL_BUILD_ENV: "${CI_REGISTRY_IMAGE}:build-environment"
  OFFICIAL_SOPHYA_ENV: "${CI_REGISTRY_IMAGE}:latest"
  SOPHYABASE: "/usr/local/SOPHYA"

12
# Docker image used for all purposes but environment building
13 14 15 16 17 18 19 20 21 22 23 24 25
default:
  image: "${BUILD_ENV}"

stages:
  - environment
  - build
  - package
  - test
  - compatibility
  - deploy
  - cleanup-sophya
  - cleanup-env

26 27 28 29 30 31 32 33 34
# Allow both push and merge request pipelines, but only run one of them: if
# a branch has opened merge requests, then stop running regular branch pipelines
# for this branch.
workflow:
  rules:
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
      when: never
    - when: always

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
# Template for jobs that build Docker images
.docker-build:
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  before_script:
    - mkdir -p /kaniko/.docker
    - |
      auth_config() {
        REGISTRY_URL="$1"
        REGISTRY_USER="$2"
        REGISTRY_PWD="$3"
        AUTH_STR=$( printf "%s:%s" "${REGISTRY_USER}" "${REGISTRY_PWD}" \
                    | base64 \
                    | tr -d '\n' )
        printf '"%s": { "auth": "%s" }' "${REGISTRY_URL}" "${AUTH_STR}"
      }
      SOPHYA_REGISTRY_CONFIG="$(auth_config ${CI_REGISTRY} ${CI_REGISTRY_USER} ${CI_REGISTRY_PASSWORD})"
      DEP_PROXY_CONFIG="$(auth_config ${CI_DEPENDENCY_PROXY_SERVER} ${CI_DEPENDENCY_PROXY_USER} ${CI_DEPENDENCY_PROXY_PASSWORD})"
      printf '{ "auths": { %s, %s } }' \
             "${SOPHYA_REGISTRY_CONFIG}" \
             "${DEP_PROXY_CONFIG}" >/kaniko/.docker/config.json


# === Set up the build environment (environment stage) ===
60

61
Setup CI build env:
62
  extends: .docker-build
63
  stage: environment
64
  interruptible: true
65
  script:
66
    - DEBUG_ENV="${CI_REGISTRY_IMAGE}:testing-build-environment"
67
    - |
68
      /kaniko/executor --build-arg CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX="${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}" \
69 70 71 72
                       --context "${CI_PROJECT_DIR}/CI" \
                       --dockerfile "${CI_PROJECT_DIR}/CI/Dockerfile.build-environment" \
                       --destination "${BUILD_ENV}" \
                       --destination "${DEBUG_ENV}"
73

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154

# === Build Sophya in C++11 and C++17 mode (build stage) ===

# Recipe to build Sophya for a given C++ standard revision
.sophya-build:
  stage: build
  interruptible: true
  script:
    # Build and install Sophya
    - cd SophyaLib/BuildMgr
    - mkdir -p ${SOPHYABASE}
    - ./configure  -sbase ${SOPHYABASE}  -scxx g++  -wextlib  -alsofftwfloat  -cxxstd ${cxxstd}
    # FIXME: In principle, we should be able to use less make statements, but in
    #        practice, the Makefile dependencies are not specified well enough.
    - make -j $(nproc) all
    - make -j $(nproc) slball
    - make -j $(nproc) prgall pmixer
    - make -j $(nproc) cleanobj

    # HACK: Gitlab artifacts must inside of the build directory, so we must
    #       move the install directory to the build directory and back :(
    - mv ${SOPHYABASE} ${CI_PROJECT_DIR}/${LOCAL_SOPHYABASE}
  artifacts:
    paths:
      - ${LOCAL_SOPHYABASE}

# Build Sophya in C++11 mode
C++11 build:
  extends: .sophya-build
  variables:
    cxxstd: "11"

# Build Sophya in C++17 mode (basis for further testing)
C++17 build:
  extends: .sophya-build
  variables:
    cxxstd: "17"


# === Deploy Sophya builds into a testing Docker image (package stage) ===

# Generic recipe for packaging a testing Sophya image for a certain C++ standard
.package-sophya-image:
  extends: .docker-build
  stage: package
  interruptible: true
  script:
    - SUFFIX="-cxx${cxxstd}"
    - |
      /kaniko/executor --build-arg BUILD_ENV="${BUILD_ENV}" \
                       --build-arg LOCAL_SOPHYABASE="${CI_PROJECT_DIR}/${LOCAL_SOPHYABASE}" \
                       --build-arg SOPHYABASE="${SOPHYABASE}" \
                       --context "/" \
                       --dockerfile "${CI_PROJECT_DIR}/CI/Dockerfile.runtime-environment" \
                       --destination "${CI_SOPHYA_ENV}${SUFFIX}" \
                       --destination "${DEBUG_SOPHYA_ENV}${SUFFIX}"

# Tag new official C++11 image
Package C++11 image:
  extends: .package-sophya-image
  variables:
    cxxstd: "11"
  needs:
    - job: "C++11 build"
      artifacts: true

# Tag new official C++11 image
Package C++17 image:
  extends: .package-sophya-image
  variables:
    cxxstd: "17"
  needs:
    - job: "C++17 build"
      artifacts: true


# === Run the tests as a child pipeline for cleaner CI UI (test stage) ===

Run tests:
  stage: test
  trigger:
155
    include: 'CI/tests.yaml'
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
    strategy: depend
  needs:
    - job: "C++17 build"
      artifacts: true
  variables:
    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
  # Inherit global variables and configuration
  inherit:
    default: true
    variables: true


# === Check that the CI of downstream projects also succeeds (compatibility stage) ===

# Commonalities between compatibility checks
# FIXME: Would like to deduplicate more, but need recursive variable expansion
.compatibility:
  inherit:
    default: false
    variables: false
  stage: compatibility

# Make sure DocSophya CI still passes with this Sophya build
DocSophya:
  extends: .compatibility
  variables:
    # FIXME: As many thing in this CI config, could use recursive expansion...
    BUILD_ENV: "${CI_REGISTRY_IMAGE}:testing-${CI_PIPELINE_IID}-cxx11"
  trigger:
    project: SOPHYA/DocSophya
    strategy: depend
  needs: ["Package C++11 image"]

# Make sure ProjectPI CI still passes with this Sophya build
ProjectPI:
  extends: .compatibility
  variables:
    # FIXME: As many thing in this CI config, could use recursive expansion...
    BUILD_ENV: "${CI_REGISTRY_IMAGE}:testing-${CI_PIPELINE_IID}-cxx11"
  trigger:
    project: SOPHYA/ProjectPI
    strategy: depend
  needs: ["Package C++11 image"]

# Make sure TAcq CI still passes with this Sophya build
TAcq:
  extends: .compatibility
  variables:
    # FIXME: As many thing in this CI config, could use recursive expansion...
    BUILD_ENV: "${CI_REGISTRY_IMAGE}:testing-${CI_PIPELINE_IID}-cxx17"
  trigger:
    project: BAORadio/TAcq
    strategy: depend
  needs: ["Package C++17 image"]


# === Once all checks passed, deploy updated Sophya images (deploy stage) ===
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
213

214
# Publish all testing images produced by CI as official Sophya image.
215
#
216
# These jobs are restricted to the develop branch of the official Sophya
217
# repository, which warrants some explanations:
218 219 220
# - We want this to only run on the develop branch, because it must be
#   compatible with the develop branch of other repositories, which may depend
#   on changes from the Sophya develop branch.
221 222 223 224
# - We want this to only run on official repositories, since this is a
#   prerequisite in order to have permissions to push to the official Sophya
#   container registry.
#
225 226
# All images are pushed by a single job in order to reduce the odds that only a
# subset of images could be updated due to errors in this process, leading to
Hadrien G's avatar
Hadrien G committed
227
# inconsistent container registry contents.
228 229
#
Deploy docker images:
230
  extends: .docker-build
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
231
  stage: deploy
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
232
  interruptible: false
233
  resource_group: deploy-docker-images
234
  script:
235
    # Define source and destination images, using C++17 image as latest image
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
236
    # NOTE: Can't use arrays here, this image doesn't use the bash shell...
237
    - OFFICIAL_IMAGES="${OFFICIAL_BUILD_ENV} ${OFFICIAL_SOPHYA_ENV}-cxx11 ${OFFICIAL_SOPHYA_ENV}-cxx17 ${OFFICIAL_SOPHYA_ENV}"
238
    - CI_IMAGES="${BUILD_ENV} ${CI_SOPHYA_ENV}-cxx11 ${CI_SOPHYA_ENV}-cxx17 ${CI_SOPHYA_ENV}-cxx17"
239
    - NUM_IMAGES=$(echo ${OFFICIAL_IMAGES} | wc -w)
240

241
    # Tag official images
242
    - |
Hadrien G's avatar
Hadrien G committed
243
      for i in $(seq 1 ${NUM_IMAGES}); do
244 245
        OFFICIAL_IMAGE=$(echo ${OFFICIAL_IMAGES} | cut -d ' ' -f $i) \
        && CI_IMAGE=$(echo ${CI_IMAGES} | cut -d ' ' -f $i) \
Hadrien G's avatar
Hadrien G committed
246
        && echo "=== Tagging CI image ${CI_IMAGE} as official image ${OFFICIAL_IMAGE}... ===" \
247
        && /kaniko/executor --build-arg SOURCE="${CI_IMAGE}" \
248 249 250
                            --context "${CI_PROJECT_DIR}/CI" \
                            --dockerfile "${CI_PROJECT_DIR}/CI/Dockerfile.retag" \
                            --destination "${OFFICIAL_IMAGE}"
251
      done
252
  rules:
253
    - if: ($CI_PROJECT_PATH == "SOPHYA/Sophya") && ($CI_COMMIT_REF_NAME == "develop") && ($CI_PIPELINE_SOURCE != "merge_request_event")
254

255

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
# === Whether CI succeeded or not, discard artifacts that won't be reused (cleanup stage) ===

# Recipe to auto-delete a Docker tag at the end of a pipeline.
#
# The tag that gets deleted is controlled by the value of the ${tag} variable.
#
# This job will run whether the pipeline suceeded or not, but may be canceled by
# a newer pipeline, so setting up container auto-deletion is still recommended.
#
# To run this CI Job, a repository must have a Project Access Token with API
# access configured as the API_TOKEN CI/CD variable. Said CI/CD variable should
# be masked since rogue bots can wreak total havoc inside of the project with
# this token if it finds its way into the outside world via a mere $(env) call.
#
# Also, in order to keep CI code simple, this code currently only works within
# projects that have a single Docker image.
#
.cleanup-docker-image:
  interruptible: false
  script:
    - |
      if [[ -z ${tag} ]]; then
        echo "No target tag specified, please set the 'tag' variable!"
        exit 1
      fi
    - REPOSITORIES_API_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories"
    - REPOSITORY_ID=$(curl ${REPOSITORIES_API_URL} | grep -Po '"id":\K([0-9]+)')
    - |
      if [[ $(echo "${REPOSITORY_ID}" | wc -l) != 1 ]]; then
        echo "ERROR: This project has >1 container repository and I don't know how to choose!"
        exit 1
      fi
    # NOTE: Every repository must have a Project Access Token with API access
    #       configured as the API_TOKEN CI/CD variable.
    #       We cannot use the CI_JOB_TOKEN mechanism as the ci_job_token_scope
    #       feature is not enabled on CCIN2P3's GitLab instance...
    - 'DELETE_REPLY=$(curl --request DELETE --header "PRIVATE-TOKEN: ${API_TOKEN}" "${REPOSITORIES_API_URL}/${REPOSITORY_ID}/tags/${tag}")'
    - |
      if [[ "${DELETE_REPLY}" != 200 ]]; then
        echo "Repository cleanup yielded unexpected result ${DELETE_REPLY}"
        exit 1
      fi
  when: always

# Clean up testing Sophya images
# FIXME: I'd like to factor this out, but the lack of recursive variable
#        expansion on this Gitlab instance won't let me...
Discard CI C++11 image:
  stage: cleanup-sophya
  extends: .cleanup-docker-image
  variables:
    tag: "testing-${CI_PIPELINE_IID}-cxx11"
#
Discard CI C++17 image:
  stage: cleanup-sophya
  extends: .cleanup-docker-image
  variables:
    tag: "testing-${CI_PIPELINE_IID}-cxx17"
314

315
# Automatically delete CI build environment image at the end of the pipeline
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
316 317 318 319
#
# This must be the last Docker job, all subsequent ones will fail because the
# build environment image is gone. Hence the dedicated cleanup-env stage.
#
320
Discard CI build env:
321
  extends: .cleanup-docker-image
GRASLAND Hadrien's avatar
GRASLAND Hadrien committed
322
  stage: cleanup-env
323
  variables:
Hadrien G's avatar
Hadrien G committed
324
    # FIXME: Redundancy imposed by lack of recursive variable expansion
325
    tag: "testing-${CI_PIPELINE_IID}-build-environment"