variables:
  DOCKER_OPT_VOLUMES: -v /hpc/local:/hpc/local -v /auto/sw_tools:/auto/sw_tools
  DOCKER_OPT_IB: --ulimit memlock=-1:-1 --device=/dev/infiniband/
  DOCKER_OPT_GPU: --gpus all $(DOCKER_OPT_IB)
  DOCKER_OPT_ARGS: --cap-add=SYS_PTRACE

resources:
  containers:
    - container: centos7
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7:4
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: centos7_ib
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7:5
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_IB) --net host
    - container: centos7_cuda11
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7-mofed5.4-cuda11:1
      options: $(DOCKER_OPT_ARGS) --net=host $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
    - container: fedora
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/fedora33:1
      options: $(DOCKER_OPT_ARGS)
    - container: fedora34
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/fedora34:2
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: coverity_rh7
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/coverity:mofed-5.1-2.3.8.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel76
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel7.6/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel76_mofed47
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel7.6/builder:mofed-4.7-1.0.0.1
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel74
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel7.4/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel72
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel7.2/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel82
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel8.2/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: rhel90
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel9.0/builder:mofed-5.6-0.5.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: ubuntu2004
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu20.04/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: ubuntu2204
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu22.04/builder:mofed-5.7-0.2.3.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: ubuntu2210
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu22.10/builder:mofed-5.8-0.2.1.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: ubuntu1804
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu18.04/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: sles15sp2
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/sles15sp2/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: sles12sp5
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/sles12sp5/builder:mofed-5.0-1.0.0.0
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
    - container: centos7_cuda_11_0
      image: nvidia/cuda:11.0.3-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_11_1
      image: nvidia/cuda:11.1.1-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_11_2
      image: nvidia/cuda:11.2.2-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_11_3
      image: nvidia/cuda:11.3.1-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_11_4
      image: nvidia/cuda:11.4.3-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda_11_0
      image: nvidia/cuda:11.0.3-devel-centos8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda_11_1
      image: nvidia/cuda:11.1.1-devel-centos8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda_11_2
      image: nvidia/cuda:11.2.2-devel-centos8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda_11_3
      image: nvidia/cuda:11.3.1-devel-centos8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda_11_4
      image: nvidia/cuda:11.4.3-devel-centos8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubi8_cuda_11_5
      image: nvidia/cuda:11.5.2-devel-ubi8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubi8_cuda_11_6
      image: nvidia/cuda:11.6.2-devel-ubi8
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_0
      image: nvidia/cuda:11.0.3-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_1
      image: nvidia/cuda:11.1.1-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_2
      image: nvidia/cuda:11.2.2-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_3
      image: nvidia/cuda:11.3.1-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_4
      image: nvidia/cuda:11.4.3-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_5
      image: nvidia/cuda:11.5.2-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_11_6
      image: nvidia/cuda:11.6.2-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_0
      image: nvidia/cuda:11.0.3-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_1
      image: nvidia/cuda:11.1.1-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_2
      image: nvidia/cuda:11.2.2-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_3
      image: nvidia/cuda:11.3.1-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_4
      image: nvidia/cuda:11.4.3-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_5
      image: nvidia/cuda:11.5.2-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_11_6
      image: nvidia/cuda:11.6.2-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_12_0
      image: nvidia/cuda:12.0.0-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos7_cuda_12_1
      image: nvidia/cuda:12.1.0-devel-centos7
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_12_0
      image: nvidia/cuda:12.0.0-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda_12_1
      image: nvidia/cuda:12.1.0-devel-ubuntu18.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_12_0
      image: nvidia/cuda:12.0.0-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda_12_1
      image: nvidia/cuda:12.1.0-devel-ubuntu20.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu22_cuda_12_0
      image: nvidia/cuda:12.0.0-devel-ubuntu22.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: ubuntu22_cuda_12_1
      image: nvidia/cuda:12.1.0-devel-ubuntu22.04
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_GPU)
    - container: centos8_cuda11
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos8-mofed5-cuda11:1
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
    - container: ubuntu16_cuda11
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu16.04-mofed5-cuda11:1
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
    - container: ubuntu18_cuda11
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu18.04-mofed5-cuda11:1
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
    - container: ubuntu20_cuda11
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu20.04-mofed5-cuda11:1
      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
    - container: ubuntu2004_rocm_5_4_0
      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu2004:rocm_5_4_0
      options: $(DOCKER_OPT_ARGS)

stages:
  - stage: Codestyle
    jobs:
    - template: codestyle.yml

  - stage: Static_check
    dependsOn: [Codestyle]
    jobs:
    - template: static_checks.yml

  - stage: Build
    dependsOn: [Static_check]
    jobs:
      - job: build_source
        pool:
          name: MLNX
          demands:
            - ucx_docker -equals yes
        strategy:
          matrix:
            rhel72:
              CONTAINER: rhel72
            rhel74:
              CONTAINER: rhel74
            rhel76:
              CONTAINER: rhel76
              long_test: yes
            rhel76_mofed47:
              CONTAINER: rhel76_mofed47
              long_test: yes
            ubuntu2004:
              CONTAINER: ubuntu2004
              long_test: yes
              extra_modules: ""
            ubuntu1804:
              CONTAINER: ubuntu1804
              extra_modules: ""
            ubuntu2204:
              CONTAINER: ubuntu2204
            ubuntu2210:
              CONTAINER: ubuntu2210
            sles15sp2:
              CONTAINER: sles15sp2
            rhel82:
              CONTAINER: rhel82
            rhel90:
              CONTAINER: rhel90
            fedora34:
              CONTAINER: fedora34
              long_test: yes
            centos7:
              CONTAINER: centos7_ib
            ubuntu2004_rocm:
              CONTAINER: ubuntu2004_rocm_5_4_0
        container: $[ variables['CONTAINER'] ]
        timeoutInMinutes: 340

        steps:
          - checkout: self
            clean: true
            fetchDepth: 100

          - bash: |
              ./buildlib/tools/builds.sh
            displayName: Build
            env:
              BUILD_ID: "$(Build.BuildId)-$(Build.BuildNumber)"
              long_test: $(long_test)
              test_static: $(test_static)

  - stage: Distro
    dependsOn: [Static_check]
    jobs:
    - template: distro.yml

  - stage: Coverity
    dependsOn: [Static_check]
    jobs:
      - template: coverity.yml
        parameters:
          demands: ucx_docker -equals yes
          container: coverity_rh7

  - stage: Tests
    dependsOn: [Static_check]
    jobs:
    - template: tests.yml
      parameters:
        name: althca
        demands: ucx_althca -equals yes
        test_perf: 0
    - template: tests.yml
      parameters:
        name: gpu
        demands: ucx_gpu -equals yes
        test_perf: 1
        container: centos7_cuda11
    - template: tests.yml
      parameters:
        name: new
        demands: ucx_new -equals yes
        test_perf: 1
    - template: tests.yml
      parameters:
        name: roce
        demands: ucx_roce -equals yes
        test_perf: 0
    - template: tests.yml
      parameters:
        name: roce_proto_enable
        demands: ucx_roce -equals yes
        test_perf: 0
        proto_enable: yes
    - template: tests.yml
      parameters:
        name: BlueField
        demands: ucx_bf -equals yes
        test_perf: 0

  - stage: io_demo
    dependsOn: [Static_check]
    jobs:
    - template: io_demo/io-demo.yml

  - stage: jucx
    dependsOn: [Static_check]
    jobs:
      - template: ../jucx/jucx-test.yml
        parameters:
          name: new
          demands: ucx_new -equals yes
      - template: ../jucx/jucx-test.yml
        parameters:
          name: gpu
          demands: ucx_gpu -equals yes

  - stage: go
    dependsOn: [Static_check]
    jobs:
      - template: go/go-test.yml
        parameters:
          name: new
          demands: ucx_new -equals yes
      - template: go/go-test.yml
        parameters:
            name: gpu
            demands: ucx_gpu -equals yes

  - stage: Build_Static
    dependsOn: [Static_check]
    jobs:
      - job: build_source
        pool:
          name: MLNX
          demands:
            - ucx_docker -equals yes
        strategy:
          matrix:
            centos7:
              CONTAINER: centos7_ib
              extra_modules: ucx-ib ucx-cma ucx-rdmacm
              extra_tls: dc_mlx5 rc_mlx5 ud_mlx5 rc_verbs ud_verbs cma
              run_tls: ib rc rc_v rc_x dc dc_x ud ud_v ud_x shm sm
            ubuntu2004:
              CONTAINER: ubuntu2004
              extra_modules: ""
              extra_tls: ""
              run_tls: ""
            ubuntu1804:
              CONTAINER: ubuntu1804
              extra_modules: ""
              extra_tls: ""
              run_tls: ""
        container: $[ variables['CONTAINER'] ]
        timeoutInMinutes: 340

        steps:
          - checkout: self
            clean: true
            fetchDepth: 100

          - bash: |
              ./buildlib/tools/build_static.sh
            displayName: Build
            env:
              EXTRA_TLS: $(extra_tls)
              RUN_TLS: $(run_tls)
              EXTRA_MODULES: $(extra_modules)
              EXECUTOR_NUMBER: $(AZP_AGENT_ID)


  - stage: Cuda
    dependsOn: [Static_check]
    jobs:
      - template: cuda/cuda.yml

#  - stage: Cuda_compatible
#    dependsOn: [Static_check]
#    jobs:
#    - template: cuda/cuda_compatible.yml
