image: quay.io/ucsc_cgl/toil_ci_prebake:latest
# Note that we must run in a privileged container for our internal Docker daemon to come up.

variables:
  PYTHONIOENCODING: "utf-8"
  DEBIAN_FRONTEND: "noninteractive"
  TOIL_OWNER_TAG: "shared"
  TOIL_HISTORY: "0"
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  MAIN_PYTHON_PKG: "python3.13"
  # Used to tell pytest which tests to be run by specifying markers,
  # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
  # Currently specifies special tests that are not run by quick_test_offline.
  MARKER: "(tes or integrative or encryption or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
  TEST_THREADS: "4"
before_script:
  # Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
  - ip addr
  # Configure Docker and Buildkit to use a mirror for Docker Hub and restart the daemon
  # Set the registry as insecure because it is probably cluster-internal over plain HTTP.
  - |
    if [[ ! -z "${DOCKER_HUB_MIRROR}" ]] ; then
        echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
        export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
        echo "[registry.\"docker.io\"]" >buildkitd.toml
        echo "  mirrors = [\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
        echo "[registry.\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
        echo "  http = true" >>buildkitd.toml
    else
        echo "" >buildkitd.toml
    fi
  # Restart or start the Docker daemon
  - stopdocker || true
  - sudo rm -f /var/run/docker.sock
  - startdocker || true
  - docker info
  - cat /etc/hosts
  - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config
  - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials
  # We need to make sure docker buildx create can't see the ~/.kube/config that we deploy. It has
  # a service account bearer token for auth and triggers https://github.com/docker/buildx/issues/267
  # where buildx can't use a bearer token from a kube config and falls back to anonymous instead
  # of using the system's service account.
  - if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then KUBECONFIG=/dev/null docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else cat buildkitd.toml ; docker buildx create --use --name=container-builder --driver=docker-container --config ./buildkitd.toml ; fi
  # Report on the builders, and make sure they exist.
  - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
  # This will hang if we can't talk to the builder
  - (echo "y" | docker buildx prune --keep-storage 80G) || true

after_script:
  # We need to clean up any files that Toil may have made via Docker that
  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
  # and clean them up before running the next job on the runner, fail, and fail
  # that next job.
  - pwd
  - sudo rm -rf tmp
  - stopdocker || true

stages:
  - linting_and_dependencies
  - basic_tests
  - main_tests
  - integration

lint:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
      - .mypy_cache
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all dependencies (except htcondor)"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
    ${MAIN_PYTHON_PKG} -m pip freeze
    ${MAIN_PYTHON_PKG} --version
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:mypy\r\e[0KCheck the Python types with mypy"
    make mypy
    echo -e "\e[0Ksection_end:`date +%s`:mypy\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:docs\r\e[0KBuild the docs"
    make docs
    echo -e "\e[0Ksection_end:`date +%s`:docs\r\e[0K"
    check-jsonschema --schemafile https://json.schemastore.org/dependabot-2.0.json .github/dependabot.yml
    # make diff_pydocstyle_report

# We make sure to also lint with our oldest supported Python version on every PR.
py39_lint:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  cache:
    key: cache-python3.9
    paths:
      - .cache/pip
      - .mypy_cache
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all dependencies (except htcondor)"
    python3.9 -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
    python3.9 -m pip freeze
    python3.9 --version
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:mypy\r\e[0KCheck the Python types with mypy"
    make mypy
    echo -e "\e[0Ksection_end:`date +%s`:mypy\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:docs\r\e[0KBuild the docs"
    make docs
    echo -e "\e[0Ksection_end:`date +%s`:docs\r\e[0K"
    check-jsonschema --schemafile https://json.schemastore.org/dependabot-2.0.json .github/dependabot.yml
    # make diff_pydocstyle_report

cwl_dependency_is_stand_alone:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:$(date +%s):prepare\r\e[0KDownload and install the CWL dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
    echo -e "\e[0Ksection_end:$(date +%s):prepare\r\e[0K"
    echo -e "\e[0Ksection_start:$(date +%s):test\r\e[0KRun a single CWL test: test_run_revsort"
    make test threads="1" tests=src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_revsort
    echo -e "\e[0Ksection_end:$(date +%s):test\r\e[0K"

wdl_dependency_is_stand_alone:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install the WDL dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun a single WDL test: test_run_MD5sum"
    make test threads="1" tests=src/toil/test/wdl/wdltoil_test.py::TestWDL::test_MD5sum
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

quick_test_offline:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: basic_tests
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    ${MAIN_PYTHON_PKG} -m virtualenv venv
    . venv/bin/activate
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install the aws, google, and wdl dependencies"
    pip install -U pip wheel
    make prepare
    make develop extras=[aws,google,wdl]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the offline tests"
    # Make sure not to run the doctests here because we didn't install all possible extras
    TOIL_TEST_QUICK=True make test_offline tests=src/toil/test threads="${TEST_THREADS}"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

py39_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.9.*/
  stage: basic_tests
  cache:
    key: cache-python3.9
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
    python setup_gitlab_docker.py
    make push_docker
    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"

py39_main:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.9.*/
  stage: basic_tests
  cache:
    key: cache-python3.9
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
    make doctest threads="${TEST_THREADS}"
    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

py310_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.10.*/
  stage: basic_tests
  cache:
    key: cache-python3.10
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
    python setup_gitlab_docker.py
    make push_docker
    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"

py310_main:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.10.*/
  stage: basic_tests
  cache:
    key: cache-python3.10
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
    make doctest threads="${TEST_THREADS}"
    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

py311_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.11.*/
  stage: basic_tests
  cache:
    key: cache-python3.11
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
    python setup_gitlab_docker.py
    make push_docker
    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"

py311_main:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.11.*/
  stage: basic_tests
  cache:
    key: cache-python3.11
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
    make doctest threads="${TEST_THREADS}"
    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

py312_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.12.*/
  stage: basic_tests
  cache:
    key: cache-python3.12
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
    python setup_gitlab_docker.py
    make push_docker
    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"

py312_main:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH =~ /.*3\.12.*/
  stage: basic_tests
  cache:
    key: cache-python3.12
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
    make doctest threads="${TEST_THREADS}"
    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

py313_appliance_build:
  stage: basic_tests
  cache:
    key: cache-python3.13
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
    python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
    python setup_gitlab_docker.py
    make push_docker
    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"

py313_main:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: basic_tests
  cache:
    key: cache-python3.13
    paths:
      - .cache/pip
  script: |
    pwd
    # todo: htcondor is not out for python 3.13 yet, we don't actively test htcondor batchsystems but should still test an htcondor install
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, except htcondor"
    python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 && pip install -U pip wheel && make prepare && make develop extras=[all]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
    make doctest threads="${TEST_THREADS}"
    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

slurm_test:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - cd contrib/slurm-test/
    - docker compose version
    - ./slurm_test.sh

cwl_v1.2:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/cwl/*'
          - 'src/toil/test/cwl/*'
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install cwl & aws dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    # Run CWL integration tests excluded from cwl_misc
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests excluded from cwl_misc"
    time make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'integrative and not conformance'"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
  artifacts:
    reports:
      junit: "*.junit.xml"
    paths:
    - "*.junit.xml"
    when: always
    expire_in: 14 day

cwl_badge:
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/cwl/*'
          - 'src/toil/test/cwl/*'
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install cwl & aws dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:download_tests\r\e[0KDownload the CWL conformance tests"
    make download_cwl_spec
    rm -rf badges1.2
    echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
  allow_failure: true
  artifacts:
    paths:
    - "badges1.2"
    - "*.junit.xml"
    reports:
      junit: "*.junit.xml"
    when: always
    expire_in: 365 day

cwl_badge_release:
  # For releases, keep the conformance badges indefinitely and not the JUnit files.
  # Everything else is same as cwl_badge
  rules:
    - if: $CI_COMMIT_TAG
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install cwl & aws dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:download_tests\r\e[0KDownload the CWL conformance tests"
    make download_cwl_spec
    rm -rf badges1.2
    echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
  allow_failure: true
  artifacts:
    paths:
    - "badges1.2"
    when: always
    expire_in: never

cwl_on_arm:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/cwl/*'
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    # Run CWL conformance tests, on an ARM cluster on AWS, using the file store
    - make test threads="${TEST_THREADS}" tests=src/toil/test/provisioners/clusterTest.py::CWLOnARMTest
  artifacts:
    reports:
      junit: "*.junit.xml"
    paths:
    - "*.junit.xml"
    when: always
    expire_in: 14 day

cwl_misc:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: main_tests
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install cwl & aws dependencies"
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the miscellaneous CWL tests"
    make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'not integrative and not conformance'"
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

#cwl_v1.2_kubernetes:
#  stage: main_tests
#  script:
#    - pwd
#    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
#    - export TOIL_KUBERNETES_OWNER=toiltest
#    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
#    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
#    - export TOIL_WORKDIR=/var/lib/toil
#    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
#    - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
#    - mkdir -p ${TOIL_WORKDIR}
#    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::TestCWLv12::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv12::test_kubernetes_cwl_conformance_with_caching"
#  artifacts:
#    reports:
#      junit: "*.junit.xml"
#    paths:
#      - "*.junit.xml"
#    when: always
#    expire_in: 14 days

wdl:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script: |
    pwd
    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install default-jew, and all the Python dependencies except htcondor"
    apt update && apt install -y default-jre
    ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the WDL tests"
    make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/
    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"

jobstore:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"

provisioner:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"

# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.

jobstore_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py"

server_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/server/*'
          - 'src/toil/test/server/*'
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    # Test server and its integration with AWS
    - make test threads="${TEST_THREADS}" tests="src/toil/test/server -k 'integrative'"

provisioner_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - python setup_gitlab_docker.py
    - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::TestUtils::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::TestAWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/wdl/wdltoil_test_kubernetes.py::WDLKubernetesClusterTest"
#    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run

google_jobstore:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - export TOIL_TEST_INTEGRATIVE=True
    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
    - export TOIL_GOOGLE_KEYNAME=id_rsa
    - export TOIL_GOOGLE_PROJECTID=toil-dev
    - make test threads="${TEST_THREADS}" tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest

mesos:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/test/mesos/*'
          - 'src/toil/batchSystems/mesos/*'
  stage: integration
  cache:
    key: cache-python3.10
    paths:
      - .cache/pip
  script:
    - pwd
    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos,google,aws]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
    - export TOIL_GOOGLE_KEYNAME=id_rsa
    - export TOIL_GOOGLE_PROJECTID=toil-dev
    - make test threads="${TEST_THREADS}" tests="src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest src/toil/test/sort/sortTest.py::SortTest::testAwsMesos src/toil/test/sort/sortTest.py::SortTest::testFileMesos src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance_with_caching src/toil/test/src/promisedRequirementTest.py::TestMesosPromisedRequirements src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"

batchsystem:
  rules:
      - if: $CI_PIPELINE_SOURCE == "schedule"
      - if: $CI_COMMIT_TAG
      - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
      - if: $CI_COMMIT_BRANCH
        changes:
          compare_to: 'refs/heads/master'
          paths:
            - 'src/toil/test/batchSystems/test_gridengine.py'
            - 'src/toil/batchSystems/gridengine.py'
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" tests=src/toil/test/batchSystems/test_gridengine.py::GridEngineTest

# Cactus-on-Kubernetes integration (as a script and not a pytest test)
cactus_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/test/cactus/test_cactus_integration.py'
  stage: integration
  cache:
    key: cache-$MAIN_PYTHON_PKG
    paths:
      - .cache/pip
  script:
    - export CACTUS_COMMIT_SHA=03295e9af99e2e9168ccd02e78a9f4c0c8dcd490
    - set -e
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test threads="${TEST_THREADS}" tests=src/toil/test/cactus/test_cactus_integration.py