pipeline.macos.yml 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. common: &common
  2. artifact_paths:
  3. - "/tmp/bazel_event_logs/*"
  4. env:
  5. BUILDKITE: "true"
  6. CI: "true"
  7. PYTHON: "3.6"
  8. RAY_USE_RANDOM_PORTS: "1"
  9. RAY_DEFAULT_BUILD: "1"
  10. LC_ALL: en_US.UTF-8
  11. LANG: en_US.UTF-8
  12. prelude_commands: &prelude_commands |-
  13. ./ci/travis/upload_build_info.sh
  14. (which bazel && bazel clean) || true;
  15. . ./ci/travis/ci.sh init && source ~/.zshrc
  16. . ./ci/travis/ci.sh build
  17. ./ci/travis/install-dependencies.sh
  18. epilogue_commands: &epilogue_commands |-
  19. # Cleanup runtime environment to save storage
  20. rm -rf /tmp/ray
  21. # Cleanup local caches (this shouldn't clean up global disk cache)
  22. bazel clean
  23. steps:
  24. - label: ":mac: :apple: Wheels and Jars"
  25. <<: *common
  26. commands:
  27. # Cleanup environments
  28. - ./ci/travis/upload_build_info.sh
  29. - (which bazel && bazel clean) || true
  30. # TODO(simon): make sure to change both PR and wheel builds
  31. # Special setup for jar builds (will be installed to the machine instead)
  32. # - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
  33. # - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
  34. - diskutil list external physical
  35. - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
  36. - java -version
  37. # Build wheels
  38. - export MAC_WHEELS=1
  39. - export MAC_JARS=1
  40. - export RAY_INSTALL_JAVA=1
  41. - . ./ci/travis/ci.sh init && source ~/.zshrc
  42. - . ./ci/travis/ci.sh build
  43. # Test wheels
  44. - . ./ci/travis/ci.sh test_wheels
  45. # Build jars
  46. - bash ./java/build-jar-multiplatform.sh darwin
  47. # Upload the wheels and jars
  48. # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
  49. - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
  50. - pip install -q docker aws_requests_auth boto3
  51. # Upload to branch directory.
  52. - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
  53. - python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
  54. # Upload to latest directory.
  55. - if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
  56. - if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
  57. - label: ":mac: :apple: Ray C++, Java and Libraries"
  58. <<: *common
  59. commands:
  60. - export RAY_INSTALL_JAVA=1
  61. - *prelude_commands
  62. - TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
  63. # Use --dynamic_mode=off until MacOS CI runs on Big Sur or newer. Otherwise there are problems with running tests
  64. # with dynamic linking.
  65. - bazel test --config=ci --dynamic_mode=off --test_env=CI $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-post_wheel_build --
  66. //:all python/ray/serve/... python/ray/dashboard/... -rllib/... -core_worker_test
  67. # clang-format is needed by java/test.sh
  68. - pip install clang-format==12.0.1
  69. - ./java/test.sh
  70. - *epilogue_commands
  71. - label: ":mac: :apple: Worker"
  72. <<: *common
  73. commands:
  74. - *prelude_commands
  75. - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
  76. - ./ci/travis/ci.sh test_cpp
  77. - label: ":mac: :apple: Small & Client"
  78. <<: *common
  79. commands:
  80. - *prelude_commands
  81. - bazel test $(./scripts/bazel_export_options) --config=ci
  82. --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX
  83. --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI
  84. --test_tag_filters=client_tests,small_size_python_tests
  85. -- python/ray/tests/...
  86. - *epilogue_commands
  87. - label: ":mac: :apple: Large"
  88. <<: *common
  89. parallelism: 3
  90. commands:
  91. - *prelude_commands
  92. - . ./ci/travis/ci.sh test_large
  93. - *epilogue_commands
  94. - label: ":mac: :apple: Medium A-J"
  95. <<: *common
  96. commands:
  97. - *prelude_commands
  98. - bazel test --config=ci $(./scripts/bazel_export_options) --test_env=CI
  99. --test_tag_filters=-kubernetes,medium_size_python_tests_a_to_j
  100. python/ray/tests/...
  101. - *epilogue_commands
  102. - label: ":mac: :apple: Medium K-Z"
  103. <<: *common
  104. commands:
  105. - *prelude_commands
  106. - bazel test --config=ci $(./scripts/bazel_export_options) --test_env=CI
  107. --test_tag_filters=-kubernetes,medium_size_python_tests_k_to_z
  108. python/ray/tests/...
  109. - *epilogue_commands