Gitlab CI/CD for Openshit with Minio Cache support, Grype Scan and Sonar linting

.gitlab-ci-base.yml

variables:
  GATEWAY:  gw
  SERVICE_1:  service_1
  SERVICE_2: service_2
  SUB_PROJECT_NAME: $SUB_PROJECT_NAME
  LIST_OF_ALL_MODULES: ${GATEWAY} ${SERVICE_1} ${SERVICE_2}
  LIST_OF_DEPLOY_MODULES: ${SERVICE_1} ${SERVICE_2}
  GRADLE_USER_HOME: ${CI_PROJECT_DIR}/.gradle

stages:
  - "assemble"
  - "test"
  - "linting"
  - "build"
  - "registry-access"
  - "prepare-image"
  - "security"
  - "deploy"

include:
  - local: assemble/app/assemble.yml
  - local: test/app/test.yml
  - local: linting/app/**.yml
  - local: build/app/build.yml
  - local: registry-access/app/registry-access.yml
  - local: prepare-image/app/prepare-image.yml
  - local: security/app/image-scan.yml
  - local: vault/vault-dev.yml
  - local: deploy/deploy_hidden.yml
  - local: deploy/deploy_develop.yml

.gitlab-ci.yml

include:
  - project: sample-project/sample-project-gitlab-ci
    ref: sample
    file:
      - .gitlab-ci-base.yml

assemble.yml

assemble:
  stage: "assemble"
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  script:
    - >
      for submodule in $LIST_OF_ALL_MODULES; do
        if git diff HEAD~ --name-only|grep ${submodule}; then
          gradle -p ${submodule} --no-daemon --build-cache --gradle-user-home ${GRADLE_USER_HOME}/cache/ assemble
        fi;      
      done
  cache:
    key: gradle-cache-key
    paths:
      - ${GRADLE_USER_HOME}/wrapper
      - ${GRADLE_USER_HOME}/cache
      - ${CI_PROJECT_DIR}/**/build
  only:
    - main
    - develop
    - merge_requests
    - /^release/.*$/
    - /^hotfix/.*$/

build.yml

build:
  stage: "build"
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  script:
    - gradle --no-daemon --build-cache --gradle-user-home ${GRADLE_USER_HOME}/cache/ build -x test
    - mc config host add sample-project $MINIO_CI_HOST $MINIO_CI_KEY $MINIO_CI_SECRET
    - >
      for item in ${LIST_OF_DEPLOY_MODULES}; do
        mc cp ${item}/build/distributions/${item}*.tar ${REPOSITORY}/builds/$CI_COMMIT_REF_SLUG/$CI_COMMIT_SHA/
      done
  cache:
    key: gradle-cache-key
    paths:
      - ${GRADLE_USER_HOME}/wrapper
      - ${GRADLE_USER_HOME}/cache
  only:
    - main
    - develop
    - /^release/.*$/
    - /^hotfix/.*$/

deploy-develop.yml

deploy_service_1:
  extends: .deploy_app
  variables:
    PROJECT: sample-project-dev-1
    APPLICATION: ${SERVICE_1}
    ENV: dev
  script:
    - oc project ${PROJECT}
    - echo $VAULT_DB_USERNAME
    - | 
      oc process -f sample-project/deploy/app/${APPLICATION}/sample-project-app-config.yml \
      --param-file=sample-project/deploy/app/${APPLICATION}/sample-project-app-config-params.yml \
      -p=ENV=${ENV} \
      -p=NAME=${APPLICATION} \
      | oc apply -f -
    - | 
      oc process -f sample-project/deploy/app/${APPLICATION}/sample-project-app-template.yml \
      --param-file=sample-project/deploy/app/${APPLICATION}/sample-project-app-template-params.yml \
      -p=ENV=${ENV} \
      -p=NAME=${APPLICATION} \
      -p=APPLICATION_PROPERTIES=${APPLICATION}-config \
      -p=OKD_NAMESPACE=${PROJECT} \
      -p=OKD_IS=${APPLICATION} \
      -p=SOURCE_REPOSITORY_REF=${CI_COMMIT_REF_SLUG} \
      -p=HASH_COMMIT=${CI_COMMIT_SHA} \
      | oc apply -f -
  when: manual
  only:
    - develop

deploy_service_2:
  extends: .deploy_app
  variables:
    PROJECT: sample-project-dev-2
    APPLICATION: ${SERVICE_2}
    ENV: dev
  script:
    - export VAULT_DB_USERNAME="$(vault kv get -field=pg_username secret/sample-project-dev-2)"
    - export VAULT_DB_PASSWORD="$(vault kv get -field=pg_password secret/sample-project-dev-2)"
    - oc project ${PROJECT}
    - echo $VAULT_DB_USERNAME
    - | 
      oc process -f sample-project/deploy/app/${APPLICATION}/sample-project-app-config.yml \
      --param-file=sample-project/deploy/app/${APPLICATION}/sample-project-app-config-params.yml \
      -p=ENV=${ENV} \
      -p=NAME=${APPLICATION} \
      -p=DB_USERNAME=$VAULT_DB_USERNAME \
      -p=DB_PASSWORD=$VAULT_DB_PASSWORD \
      | oc apply -f -
    - | 
      oc process -f sample-project/deploy/app/${APPLICATION}/sample-project-app-template.yml \
      --param-file=sample-project/deploy/app/${APPLICATION}/sample-project-app-template-params.yml \
      -p=ENV=${ENV} \
      -p=NAME=${APPLICATION} \
      -p=APPLICATION_PROPERTIES=${APPLICATION}-config \
      -p=OKD_NAMESPACE=${PROJECT} \
      -p=OKD_IS=${APPLICATION} \
      -p=SOURCE_REPOSITORY_REF=${CI_COMMIT_REF_SLUG} \
      -p=HASH_COMMIT=${CI_COMMIT_SHA} \
      | oc apply -f -
  when: manual
  only:
    - develop

deploy_hidden.yml

.deploy_app: &deploy_app
  extends: .vault_dev
  stage: deploy
  tags:
    - sample-project
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  variables:
    GIT_STRATEGY: none
    PROJECT: sample-project-development
  before_script:
    - mkdir /root/.kube && cp kube_config /root/.kube/config && chmod 600 /root/.kube/config
    - git clone -b sample-project https://gitlab-ci-token:${CI_JOB_TOKEN}@git.epam.com/sample-project/sample-project-gitlabci.git sample-project
  artifacts:
    expire_in: 1 day
    paths:
      - kube_config

docker.yml

dockerfile_all:
  stage: "linting"
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  before_script:
    - env
  script: 
    - hadolint dockerfiles/Dockerfile_*
  rules:
    - if: $CI_COMMIT_REF_SLUG == ${REF_BRANCH} 
      changes:
        - dockerfiles/*

image-scan.yml

image_scan:
  stage: "security"
  tags:
    - sample-project-ci
  image:
    name: anchore/grype:latest
    entrypoint: [ "" ]
  script:
    - grype ${IMAGE_NAME} -o table --fail-on medium
  only:
    - main
    - develop
    - /^release/.*$/
    - /^hotfix/.*$/

prepare-image.yml

prepare_images:
  stage: prepare-image
  tags:
    - sample-project-ci
  variables:
    PROJECT: sample-project-development
    GIT_STRATEGY: none
  image:
    name: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:kaniko-mc
    entrypoint: [""]
  script:
    - git clone -b sample-project https://gitlab-ci-token:${CI_JOB_TOKEN}@git.company.com/sample-project/sample-project-gitlabci.git sample-project
    - mkdir -p /kaniko/.docker && cp config.json /kaniko/.docker/config.json
    - /opt/mc config host add sample-project $MINIO_CI_HOST $MINIO_CI_KEY $MINIO_CI_SECRET
    - /opt/mc cp --recursive sample-project/builds/${CI_COMMIT_REF_SLUG}/${CI_COMMIT_SHA}/ ./
    - /opt/mc rm --force --recursive sample-project/builds/${CI_COMMIT_REF_SLUG}/${CI_COMMIT_SHA}/
    - >
      for app in ${LIST_OF_DEPLOY_MODULES}; do
        export ATATAG=$(echo ${HOST}:${PORT}/${PROJECT}/${app}:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA})
        /kaniko/executor --build-arg APP_ARG=$app --context . --dockerfile ./sample-project/dockerfiles/Dockerfile_${app} --insecure --skip-tls-verify --destination $ATATAG --cleanup
      done
  only:
    - main
    - develop
    - /^release/.*$/
    - /^hotfix/.*$/

registry-access.yml

registry_access:
  stage: registry-access
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  variables:
    GIT_STRATEGY: none
  script:
    - oc login $OPENSHIFT_SERVER_AWS --username=$OPENSHIFT_CI_USER --password=$OPENSHIFT_CI_PASSWORD -n $OPENSHIFT_CI_PROJECT --insecure-skip-tls-verify
    - echo "{\"auths\":{\"${HOST_REGISTRY}:${PORT_REGISTRY}\":{\"username\":\"$OPENSHIFT_CI_USER\",\"password\":\"$(oc whoami -t)\"}}}" > config.json
    - oc config view > kube_config
  artifacts:
    paths:
      - config.json
      - kube_config
  only:
    - main
    - develop
    - /^release/.*$/
    - /^hotfix/.*$/

sample-project-app-config-params.yml

NAME=sample-project-app
ENV=dev
JDBC_URL=jdbc:postgresql://${HOST_DB}:${PORT}/sample-project
APP_URL=http://0.0.0.0:8080

sample-project-app-config.yml

kind: Template
apiVersion: template.openshift.io/v1
labels:
  template: '${NAME}-template'
  app: '${NAME}'
  env: '${ENV}'
metadata:
  name: '${NAME}-${ENV}'
objects:
  - kind: Secret
    apiVersion: v1
    metadata:
      name: '${NAME}-${ENV}-config'
    type: Opaque
    stringData:
      application.yaml: |-
        baseAppUrl: ${APP_URL}

        sample-project-security:
          enable-make-me-feature: true

        sso:
          provider-name: provider # any kind of providers, should be replaced
          hostname: provider_host # provider host, should be replaced
          client-id: client_id # client id, should be replaced
          client-secret: client_secret # client secret, should be replaced

        spring:
          datasource:
            driver-class-name: org.postgresql.Driver
            url: ${JDBC_URL}
            username: ${DB_USERNAME}
            password: ${DB_PASSWORD}

          flyway:
            schemas: unified
            table: schema_version
            out-of-order: true
            ignore-missing-migrations: true
            locations:
              - classpath:db/migrations/2022      

          security:
            oauth2:
              resourceserver:
                jwt:
                  jwk-set-uri: https://${sso.hostname}/auth/realms/plusx/protocol/openid-connect/certs
              client:
                registration: 
                  client_name: #should be replaced
                    client-id: ${sso.client-id}
                    client-secret: ${sso.client-secret}
                    scope: openid, profile
                    authorization-grant-type: authorization_code
                    client-authentication-method: client_secret_post
                    redirect-uri: ${baseAppUrl}/login/oauth2/code/${sso.provider-name}
                provider:
                  provider_name: #should be replaced
                    authorization-uri: https://${sso.hostname}/auth/realms/plusx/protocol/openid-connect/auth
                    token-uri: https://${sso.hostname}/auth/realms/plusx/protocol/openid-connect/token
                    jwk-set-uri: https://${sso.hostname}/auth/realms/plusx/protocol/openid-connect/certs
                    user-info-uri: https://${sso.hostname}/auth/realms/plusx/protocol/openid-connect/userinfo
                    user-info-authentication-method: header
                    user-name-attribute: sub
        management:
          endpoints:
            web:
              exposure:
                include: '*'
        logging:
          level:
            root: "debug"
            org.springframework.context: "trace"
            org.springframework.data: "trace"
            org.springframework.security: DEBUG

parameters:
  - description: Set config name to use for override spring-boot application properties.
    displayName: Application properties file
    name: NAME
    value: sample-project-app
  - description: ENV for Service
    displayName: ENV
    name: ENV
    value: dev
  - description: JDBC url connect to db
    displayName: JDBC url
    name: JDBC_URL
    value: jdbc:postgresql://${HOST_DB}:${PORT_DB}/sample-project
  - description: DB username
    displayName: DB username
    name: DB_USERNAME
    value: developer # should be replaced
  - description: DB password
    displayName: DB password
    name: DB_PASSWORD
    value: YOUR_PASSWORD # should be replaced
  - description: Application URL
    displayName: Application URL
    name: APP_URL
    value: "http://0.0.0.0:8080"

sample-project-app-template-params.yml

MEMORY_LIMIT=1024MB
MEMORY_REQUESTED=512MB
CPU_LIMIT=1000m
CPU_REQUESTED=500m
PORT="8080"
NAME=sample-project-app
REPLICA_NUMBER="1"
APPLICATION_PROPERTIES=sample-project-app
ENV=dev
JAVA_OPTIONS="-Xms512M -Xmx1000M -Duser.timezone=GMT -XX:+OptimizeStringConcat -XX:+UseCodeCacheFlushing -XX:+AlwaysActAsServerClassMachine -XX:+ExitOnOutOfMemoryError -XX:+UseContainerSupport -XX:+UseStringDeduplication -XX:+PerfDisableSharedMem -XX:MaxRAMPercentage=80 -XX:InitialRAMPercentage=50 -XX:HeapDumpPath=/opt/tmp/heapdump.bin"
OKD_NAMESPACE=sample-project-development
OKD_IS=sample-project

sample-project-app-template.yml

kind: Template
apiVersion: v1
labels:
  template: '${NAME}-template'
  app: '${NAME}'
  env: '${ENV}'
  sclale_up_down: 'yes'
  monitoring: 'yes'
metadata:
  name: '${NAME}'
objects:
  - kind: DeploymentConfig
    apiVersion: v1
    metadata:
      annotations:
        description: Defines how to deploy the application server
      name: '${NAME}'
    spec:
      replicas: ${{REPLICA_NUMBER}}
      selector:
        name: '${NAME}'
      strategy:
        type: Rolling
      template:
        metadata:
          labels:
            tagZabbix: 'true'
            env: "${ENV}"
            name: "${NAME}"
          name: "${NAME}"
        spec:
          containers:
          - env:
            # to use GC1 in java
            - name: JAVA_OPTS
              value: "${JAVA_OPTIONS}"
            image: "${HOST_REGISTRY}:${HOST_PORT}/${OKD_NAMESPACE}/${OKD_IS}:${SOURCE_REPOSITORY_REF}-${HASH_COMMIT}"
            imagePullPolicy: Always
            name: "${NAME}"
            ports:
              - containerPort: ${PORT}
            livenessProbe:
              httpGet:
                path: /actuator/health/liveness
                port: 8080
                scheme: HTTP
              initialDelaySeconds: 40
              timeoutSeconds: 5
              periodSeconds: 20
              successThreshold: 1
              failureThreshold: 5
            readinessProbe:
              httpGet:
                path: /actuator/health/readiness
                port: 8080
                scheme: HTTP
              initialDelaySeconds: 40
              timeoutSeconds: 5
              periodSeconds: 15
              successThreshold: 1
              failureThreshold: 5
            resources:
              requests:
                cpu: "${CPU_REQUESTED}"
                memory: "${MEMORY_REQUESTED}"
              limits:
                cpu: "${CPU_LIMIT}"
                memory: "${MEMORY_LIMIT}"
            volumeMounts:
              - name: properties
                mountPath: /opt/app/application.yaml
                subPath: application.yaml
              
          tolerations:
          - key: nodes.company.com/zone
            operator: Equal
            value: a
            effect: NoSchedule
          - key: nodes.company.com/zone
            operator: Equal
            value: b
            effect: NoSchedule
          - key: nodes.company.com/zone
            operator: Equal
            value: c
            effect: NoSchedule
          affinity:
            nodeAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                nodeSelectorTerms:
                - matchExpressions:
                  - key: nodes.company.com/scope
                    operator: In
                    values:
                    - critical
                  - key: nodes.company.com/zone
                    operator: In
                    values:
                    - a
                    - b
                    - c
            podAntiAffinity:
              preferredDuringSchedulingIgnoredDuringExecution:
                - weight: 100
                  podAffinityTerm:
                    labelSelector:
                      matchExpressions:
                        - key: name
                          operator: In
                          values:
                            - "${NAME}"
                    topologyKey: nodes.company.com/zone
          volumes:
            - name: properties
              secret:
                secretName: '${APPLICATION_PROPERTIES}-${ENV}-config'
            
      triggers:
        - type: ConfigChange
parameters:
  - description: Maximum amount of memory the container can use.
    displayName: Memory Limit
    name: MEMORY_LIMIT
    required: true
    value: 512MB
  - description: Requested Memory during start of container
    displayName: Requested Memory
    name: MEMORY_REQUESTED
    required: true
    value: 512MB
  - description: Maximum amount of cpu the container can use.
    displayName: CPU Limit
    name: CPU_LIMIT
    required: true
    value: 1000m
  - description: Requested CPU during start of container
    displayName: Requested CPU
    name: CPU_REQUESTED
    required: true
    value: 500m
  - description: Port of the container.
    displayName: PORT of application
    name: PORT
    required: true
    value: "8080"
  - description: The name assigned to all of the frontend objects defined in this template.
    displayName: Name
    name: NAME
    required: true
    value: sample-project-app
  - description: number of replicas of app
    displayName: replica_number
    name: REPLICA_NUMBER
    required: true
    value: "1"
  - description: Set config name to use for override spring-boot application properties.
    displayName: Application properties file
    name: APPLICATION_PROPERTIES
    value: sample-project-app
  - description: Environment specificaon
    displayName: env
    name: ENV
    required: true
    value: dev
  - description: Java Options
    displayName: Java Options
    name: JAVA_OPTIONS
    value: -Xms512M -Xmx512M -Duser.timezone=GMT -XX:+OptimizeStringConcat -XX:+UseCodeCacheFlushing   
  - description: HASH_COMMIT
    displayName: HASH_COMMIT
    name: HASH_COMMIT
    required: true
    value: 59049081eb93d2eb090eddae5eef456e759c1b53
  - description: SOURCE_REPOSITORY_REF
    displayName: SOURCE_REPOSITORY_REF
    name: SOURCE_REPOSITORY_REF
    required: true
    value: develop
  - name: OKD_NAMESPACE
    required: true
    value: 'sample-project-development'
  - name: OKD_IS
    required: true
    value: sample-project

sonar.yml

sonar:
  stage: "linting"
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  variables:
    SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"
    GIT_DEPTH: "0"
  cache:
    key: gradle-cache-key
    paths:
      - ${GRADLE_USER_HOME}/wrapper
      - ${GRADLE_USER_HOME}/cache
      - ${SONAR_USER_HOME}/cache
      - ${CI_PROJECT_DIR}/**/build
  script:
    - >
      for submodule in $LIST_OF_ALL_MODULES; do
        if git diff HEAD~ --name-only|grep ${submodule}; then
          gradle -p ${submodule} --no-daemon --build-cache --gradle-user-home ${GRADLE_USER_HOME}/cache/ sonarqube -Dsonar.coverage.jacoco.xmlReportPaths=${CI_PROJECT_DIR}/${submodule}/build/reports/jacoco/test/jacocoTestReport.xml
        fi;      
      done
  only:
    - main
    - develop
    - merge_requests
    - /^release/.*$/
    - /^hotfix/.*$/

test.yml

test:
  stage: "test"
  variables:
    TEST_DATABASE_NAME: sample_project_test_${CI_JOB_ID}
    SPRING_DATASOURCE_URL: "jdbc:postgresql://${HOST_DB}:${PORT_DB}/${TEST_DATABASE_NAME}"
    SPRING_DATASOURCE_USERNAME: "developer"
    SPRING_DATASOURCE_PASSWORD: ${SPRING_DB_PASSWORD}
  tags:
    - sample-project-ci
  image: ${HOST_REGISTRY}:${PORT_REGISTRY}/sample-project-ci/sample-project-base-image:gradle-jdk17-mc
  before_script:
    - PGPASSWORD=${PG_MAIN_PASSWORD} psql -U postgres -h "${HOST_DB}" -p "${PORT_DB}" --command="CREATE DATABASE ${TEST_DATABASE_NAME} OWNER ${SPRING_DATASOURCE_USERNAME};"
  script:
    - >
      for submodule in ${LIST_OF_ALL_MODULES}; do
        if git diff HEAD~ --name-only|grep ${submodule}; then
          gradle -p ${submodule} --no-daemon --build-cache --gradle-user-home ${GRADLE_USER_HOME}/cache/ test
        fi;      
      done
  after_script:
    - PGPASSWORD=${PG_MAIN_PASSWORD} psql -U postgres -h "${HOST_DB}" -p "${PORT_DB}" --command="DROP DATABASE ${TEST_DATABASE_NAME};"
  cache:
    key: gradle-cache-key
    paths:
      - ${GRADLE_USER_HOME}/wrapper
      - ${GRADLE_USER_HOME}/cache
      - ${CI_PROJECT_DIR}/**/build
  artifacts:
    when: always
    reports:
      junit: "**/build/test-results/test/**/TEST-*.xml"
  only:
    - main
    - develop
    - /^release/.*$/
    - /^hotfix/.*$/

vault-dev.yml

.vault_dev: &vault_dev
  before_script:
    - export VAULT_ADDR=https://${VAULT_HOST}:${VAULT_PORT}
    - export VAULT_SKIP_VERIFY=true
    - export VAULT_NAMESPACE=bss-dev/sample-project
    - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=gitlab-sample-project-ci jwt=$CI_JOB_JWT)"