group: core tests
depends_on:
  - forge
  - oss-ci-base_build
steps:
  # builds
  - name: corebuild
    wanda: ci/docker/core.build.py39.wanda.yaml
    env:
      IMAGE_FROM: cr.ray.io/rayproject/oss-ci-base_build
      IMAGE_TO: corebuild
      RAYCI_IS_GPU_BUILD: "false"

  - name: coregpubuild
    wanda: ci/docker/core.build.py39.wanda.yaml
    depends_on: oss-ci-base_gpu
    env:
      IMAGE_FROM: cr.ray.io/rayproject/oss-ci-base_gpu
      IMAGE_TO: coregpubuild
      RAYCI_IS_GPU_BUILD: "true"

  - name: corebuild-multipy
    label: "wanda: corebuild-py{{matrix}}"
    wanda: ci/docker/core.build.wanda.yaml
    matrix:
      - "3.12"
    env:
      PYTHON: "{{matrix}}"
    depends_on: oss-ci-base_build-multipy

  - name: minbuild-core
    label: "wanda: minbuild-core-py{{matrix}}"
    wanda: ci/docker/min.build.wanda.yaml
    matrix:
      - "3.9"
      - "3.10"
      - "3.11"
      - "3.12"
    env:
      PYTHON_VERSION: "{{matrix}}"
      EXTRA_DEPENDENCY: core

  - wait: ~
    depends_on:
      - corebuild

  # tests
  - label: ":ray: core: python tests"
    tags:
      - python
      - dashboard
    instance_type: large
    parallelism: 4
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
        --only-tags use_all_core --skip-ray-installation

  - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
    if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
    tags:
      - python
      - dashboard
    instance_type: large
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
        --python-version {{matrix.python}}
        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
        --python-version {{matrix.python}}
        --only-tags use_all_core --skip-ray-installation
    depends_on: corebuild-multipy
    matrix:
      setup:
        python: ["3.12"]
        worker_id: ["0", "1", "2", "3"]

  - label: ":ray: core: redis tests"
    tags: python
    instance_type: large
    parallelism: 4
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
        --test-env=TEST_EXTERNAL_REDIS=1
        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
        --only-tags use_all_core --skip-ray-installation

  - label: ":ray: core: memory pressure tests"
    tags:
      - python
      - oss
      - skip-on-premerge
    instance_type: medium
    commands:
      - cleanup() { ./ci/build/upload_build_info.sh; }; trap cleanup EXIT
      - (cd python/ray/dashboard/client && npm ci && npm run build)
      - pip install -e python[client]
      - bazel test --config=ci --jobs=1 $(./ci/run/bazel_export_options)
        --test_tag_filters=mem_pressure -- //python/ray/tests/...
    job_env: corebuild

  - label: ":ray: core: out of disk tests"
    tags:
      - python
      - oss
    instance_type: small
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
        --only-tags=tmpfs --tmp-filesystem=tmpfs

  - label: ":ray: core: out of disk redis tests"
    tags:
      - python
      - oss
    instance_type: small
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
        --test-env=TEST_EXTERNAL_REDIS=1
        --only-tags=tmpfs --tmp-filesystem=tmpfs

  - label: ":ray: core: workflow tests"
    tags:
      - python
      - workflow
    instance_type: medium
    parallelism: 2
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/workflow/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
        --except-tags use_all_core
        --parallelism-per-worker 2
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/workflow/... core
        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
        --skip-ray-installation
        --only-tags use_all_core

  - label: ":ray: core: doc tests"
    tags:
      - python
      - doc
    instance_type: large
    commands:
      # doc tests
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/... //doc/... core
        --only-tags doctest
        --parallelism-per-worker 3
      # doc examples
      - bazel run //ci/ray_ci:test_in_docker -- //doc/... core
        --except-tags gpu,post_wheel_build,mem_pressure,doctest
        --parallelism-per-worker 3
        --skip-ray-installation

  - label: ":ray: core: data tests"
    tags: python
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker --
        python/ray/util/dask/... python/ray/tests/modin/... core
        --build-name datalbuild
        --parallelism-per-worker 2
    depends_on:
      - datalbuild
      - forge

  - label: ":ray: core: dashboard tests"
    tags:
      - python
      - dashboard
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... core
        --parallelism-per-worker 3
      # ui tests
      - docker run -i --rm --volume /tmp/artifacts:/artifact-mount --shm-size=2.5gb
        "$${RAYCI_WORK_REPO}":"$${RAYCI_BUILD_ID}"-corebuild /bin/bash -iecuo pipefail
        "./python/ray/dashboard/tests/run_ui_tests.sh"

  - label: ":ray: core: debug test"
    tags: python
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
        --build-type debug
        --parallelism-per-worker 3
        --only-tags debug_tests
        --except-tags kubernetes,manual

  - label: ":ray: core: asan tests"
    tags: python
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
        --build-type asan
        --parallelism-per-worker 2
        --only-tags asan_tests
        --except-tags kubernetes,manual

  - label: ":ray: core: wheel tests"
    tags: linux_wheels
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //doc/... core
        --build-type wheel
        --parallelism-per-worker 3
        --only-tags post_wheel_build
        --test-env=RAY_CI_POST_WHEEL_TESTS=True
    depends_on:
      - manylinux
      - corebuild
      - forge

  - label: ":ray: core: minimal tests {{matrix}}"
    tags:
      - python
      - dashboard
      - oss
    instance_type: medium
    commands:
      # validate minimal installation
      - python ./ci/env/check_minimal_install.py
      # core tests
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dashboard/... core
        --parallelism-per-worker 3
        --build-name minbuild-core-py{{matrix}}
        --test-env=RAY_MINIMAL=1
        --test-env=EXPECTED_PYTHON_VERSION={{matrix}}
        --only-tags minimal
        --except-tags basic_test,manual
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dashboard/... core
        --parallelism-per-worker 3
        --build-name minbuild-core-py{{matrix}}
        --test-env=RAY_MINIMAL=1
        --test-env=EXPECTED_PYTHON_VERSION={{matrix}}
        --only-tags minimal
        --except-tags no_basic_test,manual
        --skip-ray-installation
      # core redis tests
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dashboard/... core
        --parallelism-per-worker 3
        --build-name minbuild-core-py{{matrix}}
        --test-env=RAY_MINIMAL=1
        --test-env=TEST_EXTERNAL_REDIS=1
        --test-env=EXPECTED_PYTHON_VERSION={{matrix}}
        --only-tags minimal
        --except-tags no_basic_test,manual
        --skip-ray-installation
      # serve tests
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dashboard/... serve
        --parallelism-per-worker 3
        --build-name minbuild-core-py{{matrix}}
        --test-env=RAY_MINIMAL=1
        --only-tags minimal
        --skip-ray-installation
    depends_on:
      - minbuild-core
    matrix:
      - "3.9"
      - "3.10"
      - "3.11"
      - "3.12"

  # cpp tests
  - label: ":ray: core: cpp tests"
    tags: core_cpp
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type clang
        --parallelism-per-worker 2

  - label: ":ray: core: cpp asan tests"
    tags: core_cpp
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type asan-clang
        --parallelism-per-worker 2

  - label: ":ray: core: cpp ubsan tests"
    tags: core_cpp
    instance_type: large
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type ubsan
        --except-tags no_ubsan
        --parallelism-per-worker 2

  - label: ":ray: core: cpp tsan tests"
    tags: core_cpp
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type tsan-clang
        --except-tags no_tsan
        --parallelism-per-worker 2

  - label: ":ray: core: flaky tests"
    key: core_flaky_tests
    tags:
      - python
      - skip-on-premerge
    instance_type: large
    soft_fail: true
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //... core
        --run-flaky-tests  --build-type clang
        --except-tags multi_gpu

  - label: ":ray: core: flaky gpu tests"
    key: core_flaky_gpu_tests
    tags:
      - gpu
      - python
      - skip-on-premerge
    instance_type: gpu-large
    soft_fail: true
    commands:
      - bazel run //ci/ray_ci:test_in_docker -- //... core
        --run-flaky-tests  --build-type clang
        --parallelism-per-worker 2 --gpus 2
        --build-name coregpubuild
        --only-tags multi_gpu
    depends_on: coregpubuild

  - label: ":ray: core: cpp worker tests"
    tags: core_cpp
    instance_type: medium
    commands:
      - if [[ "$${BUILDKITE_PIPELINE_ID}" == "0189942e-0876-4b8f-80a4-617f988ec59b" ]]; then
          echo "build --remote_upload_local_results=false" >> ~/.bazelrc;
        fi
      - ci/ci.sh build
      - ci/ci.sh test_cpp
    depends_on: oss-ci-base_build
    job_env: oss-ci-base_build

  - label: ":ray: core: HA integration tests"
    tags:
      - python
      - docker
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu --canonical-tag ha_integration
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core --only-tags ha_integration
    depends_on:
      - manylinux
      - forge
      - raycpubase
      - corebuild

  - label: ":ray: core: container tests"
    tags:
      - python
      - docker
      - oss
    instance_type: medium
    commands:
      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu
        --canonical-tag test_container
      - docker build --progress=plain --build-arg BASE_IMAGE="rayproject/ray:test_container"
        -t rayproject/ray:runtime_env_container -f ci/docker/runtime_env_container/Dockerfile .
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core --only-tags container
    depends_on:
      - manylinux
      - forge
      - raycpubase
      - corebuild

  - label: ":ray: core: multi gpu tests"
    tags:
      - accelerated_dag
      - gpu
    instance_type: gpu-large
    commands:
      # This machine has 4 GPUs, and we need 2 GPUs, so allow 2 tests to run in
      # parallel.
      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... core
        --parallelism-per-worker 2 --gpus 2
        --build-name coregpubuild
        --only-tags multi_gpu
    depends_on: coregpubuild