- label: ":tv: :steam_locomotive: Train GPU tests "
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
    - ./ci/env/install-horovod.sh
    - ./ci/env/env_info.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...

- label: ":tv: :database: :steam_locomotive: Datasets Train Integration GPU Tests and Examples (Python 3.7)"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
    - ./ci/env/env_info.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train doc/...

- label: ":tv: :brain: RLlib: Multi-GPU Tests"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
    - ./ci/env/env_info.sh
    # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
    # and running tests in parallel would cause timeouts as the other scripts would
    # wait for the GPU to become available.
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --jobs 2
      --test_tag_filters=multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...

- label: ":tv: :airplane: AIR GPU tests (ray/air)"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_ML_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
    - ./ci/env/install-horovod.sh
    - ./ci/env/env_info.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...

- label: ":tv: :book: Doc GPU tests and examples"
  conditions:
    ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
    - ./ci/env/env_info.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-timeseries_libs,-py37,-post_wheel_build doc/...