##============================================================================
##  The contents of this file are covered by the Viskores license. See
##  LICENSE.txt for details.
##
##  By contributing to this file, all contributors agree to the Developer
##  Certificate of Origin Version 1.1 (DCO 1.1) as stated in DCO.txt.
##============================================================================

##=============================================================================
##
##  Copyright (c) Kitware, Inc.
##  All rights reserved.
##  See LICENSE.txt for details.
##
##  This software is distributed WITHOUT ANY WARRANTY; without even
##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
##  PURPOSE.  See the above copyright notice for more information.
##
##=============================================================================

default:
  id_tokens:
    OLCF_ID_TOKEN:
      aud: https://code.olcf.ornl.gov

.frontier-common:
  variables:
    CUSTOM_CI_BUILDS_DIR: "/lustre/orion/ums033/scratch/ums033_auser/ci/viskores/runtime"
    OLCF_SERVICE_ACCOUNT: "ums033_auser"
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "release"'

# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Spock.
.frontier_gcc_hip:
  extends:
    .frontier-common
  variables:
    CCACHE_BASEDIR:          "/lustre/orion/ums033/scratch/"
    CCACHE_DIR:              "/lustre/orion/ums033/scratch/ums033_auser/ci/viskores/ccache"

    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
    CCACHE_IGNOREOPTIONS:    "-isystem=*"
    CCACHE_NOHASHDIR:        "true"
    CCACHE_INSTALL_DIR:      "$CI_PROJECT_DIR/build"

    CMAKE_BUILD_TYPE:        "RelWithDebInfo"
    CMAKE_GENERATOR:         "Ninja"
    CMAKE_PREFIX_PATH:       "$CI_BUILDS_DIR/kokkos_install"

    # We do not want to use the user's ~/.gitconfig
    GIT_CONFIG_GLOBAL:       "true"
    GIT_SUBMODULE_STRATEGY: recursive

    KOKKOS_OPTS: >-
      -DCMAKE_INSTALL_PREFIX:PATH=$CI_BUILDS_DIR/kokkos_install
      -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.4.3/hip/bin/hipcc
      -DKokkos_ARCH_VEGA90A:BOOL=ON

    # The user default module list should not exist
    # craype;rocm;gcc should be loaded first
    JOB_MODULES: >-
      DefApps
      craype-accel-amd-gfx90a
      gcc/12
      cmake
      rocm/5.4.3
      git
      git-lfs
      ninja
    OLCF_SERVICE_ACCOUNT: "ums033_auser"
    VISKORES_SETTINGS: kokkos+hip+gfx90a+frontier+ccache+no_rendering
  interruptible: true

setup_modules:
  stage: .pre
  tags: [frontier, shell]
  variables:
    CUSTOM_CI_BUILDS_DIR: "/lustre/orion/ums033/scratch/ums033_auser/ci/viskores/runtime"
    OLCF_SERVICE_ACCOUNT: "ums033_auser"
    GIT_STRATEGY: none
  script:
    - module load DefApps
    - module load git-lfs
    - module save

.setup_env_ecpci: &setup_env_ecpci |
  module purge
  module load ${JOB_MODULES}
  module list
  export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"

build:frontier_gcc_hip:
  stage: build
  tags: [frontier, shell]
  extends:
    - .frontier_gcc_hip
  before_script:
    - *setup_env_ecpci
    - mkdir -p "$CCACHE_INSTALL_DIR"
    - cmake --version
    - cmake -VV -P .gitlab/ci/config/ccache.cmake
    - ccache -z
    - ccache -s
    - .gitlab/ci/config/kokkos.sh "$CI_BUILDS_DIR" "3.7.01" $KOKKOS_OPTS
  script:
    - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake
    - ctest -VV -S .gitlab/ci/ctest_configure.cmake
  artifacts:
    expire_in: 24 hours
    when: always
    paths:
      - build/
      - .gitlab/ccache/ccache

test:frontier_gcc_hip:
  stage: test
  tags: [frontier, slurm]
  extends:
    - .frontier_gcc_hip
  needs:
    - build:frontier_gcc_hip
  variables:
    # -c48 --gpus=8 since we want to use fully utilize a node
    # SMT=2 since compiling is balanced type of task (IO|computing).
    SCHEDULER_PARAMETERS: "-AUMS033 -pbatch -t 00:30:00 --nice=0 -c48 --threads-per-core=2 --gpus=8 -N 1"
    # We need this to skip ctest_submit from being run inside a jsrun job
    GITLAB_CI_EMULATION: 1
    # Tests errors to address due to different env in Spock
    # Refer to issue: https://gitlab.kitware.com/vtk/vtk-m/-/issues/743
    CTEST_EXCLUSIONS: >-
      UnitTestMIRFilter
      UnitTestDistributedBranchDecompositionFilter
      UnitTestWorkletParticleAdvection
      UnitTestLagrangianFilter
  before_script:
    - *setup_env_ecpci
  script:
    - CTEST_MAX_PARALLELISM=96 ctest -VV -S .gitlab/ci/ctest_build.cmake
    - CTEST_MAX_PARALLELISM=8  ctest -VV -S .gitlab/ci/ctest_test.cmake
  after_script:
    - *setup_env_ecpci
    - ccache -s
    - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake
    - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake
  timeout: 150 minutes

.report-status:
  tags: [ shell ]
  extends:
    .frontier-common
  variables:
    STATUS_PROJECT: Viskores/viskores
    STATUS_NAME: OLCF Frontier
  script: >
    curl -X POST -H "${GITHUB_CURL_HEADERS}"
    "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}"
    -d "{\"state\":\"${PIPELINE_STATE}\", \"context\":\"${STATUS_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${PIPELINE_DESCRITION}\"}"
  environment:
    name: report-$CI_COMMIT_SHA

pending:
  stage: .pre
  variables:
    PIPELINE_DESCRITION: Pipeline is running
    PIPELINE_STATE: pending
  extends:
    - .report-status

success:
  stage: .post
  variables:
    PIPELINE_DESCRITION: Pipeline succeeded
    PIPELINE_STATE: success
  extends:
    - .report-status
  dependencies:
    - build:frontier_gcc_hip
    - test:frontier_gcc_hip

failure:
  stage: .post
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: on_failure
    - if: '$CI_COMMIT_BRANCH == "release"'
      when: on_failure
  variables:
    PIPELINE_DESCRITION: Pipeline failed
    PIPELINE_STATE: failure
  extends:
    - .report-status
  dependencies:
    - build:frontier_gcc_hip
    - test:frontier_gcc_hip
