test_byod_build.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164
  1. import sys
  2. import pytest
  3. from unittest.mock import patch
  4. from typing import List
  5. from ray_release.bazel import bazel_runfile
  6. from ray_release.configs.global_config import init_global_config, get_global_config
  7. from ray_release.test import Test
  8. from ray_release.byod.build import (
  9. build_anyscale_custom_byod_image,
  10. build_anyscale_base_byod_images,
  11. build_champagne_image,
  12. DATAPLANE_FILENAME,
  13. _get_ray_commit,
  14. )
  15. def test_get_ray_commit() -> None:
  16. assert (
  17. _get_ray_commit(
  18. {
  19. "RAY_WANT_COMMIT_IN_IMAGE": "abc123",
  20. "COMMIT_TO_TEST": "def456",
  21. "BUILDKITE_COMMIT": "987789",
  22. }
  23. )
  24. == "abc123"
  25. )
  26. assert (
  27. _get_ray_commit(
  28. {
  29. "COMMIT_TO_TEST": "def456",
  30. "BUILDKITE_COMMIT": "987789",
  31. }
  32. )
  33. == "def456"
  34. )
  35. assert _get_ray_commit({"BUILDKITE_COMMIT": "987789"}) == "987789"
  36. assert _get_ray_commit({"PATH": "/usr/bin"}) == ""
  37. def test_build_anyscale_champagne_image() -> None:
  38. cmds = []
  39. def _mock_check_call(
  40. cmd: List[str],
  41. *args,
  42. **kwargs,
  43. ) -> None:
  44. cmds.append(cmd)
  45. with patch.dict(
  46. "os.environ",
  47. {"BUILDKITE_COMMIT": "abc123", "BUILDKITE_BRANCH": "master"},
  48. ), patch(
  49. "ray_release.byod.build._download_dataplane_build_file",
  50. return_value=None,
  51. ), patch(
  52. "subprocess.check_call",
  53. side_effect=_mock_check_call,
  54. ), patch(
  55. "subprocess.check_output",
  56. return_value=b"abc123",
  57. ), open(
  58. DATAPLANE_FILENAME, "wb"
  59. ) as _:
  60. build_champagne_image("2.5.1", "py37", "cpu")
  61. assert "docker build --build-arg BASE_IMAGE=rayproject/ray:2.5.1-py37 -t "
  62. "029272617770.dkr.ecr.us-west-2.amazonaws.com/"
  63. "anyscale/ray:champagne-2.5.1 -" == " ".join(cmds[0])
  64. init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
  65. def test_build_anyscale_custom_byod_image() -> None:
  66. cmds = []
  67. def _mock_check_call(
  68. cmd: List[str],
  69. *args,
  70. **kwargs,
  71. ) -> None:
  72. cmds.append(cmd)
  73. with patch("ray_release.byod.build._image_exist", return_value=False), patch.dict(
  74. "os.environ",
  75. {"BUILDKITE_COMMIT": "abc123", "BUILDKITE_BRANCH": "master"},
  76. ), patch("subprocess.check_call", side_effect=_mock_check_call,), patch(
  77. "subprocess.check_output",
  78. return_value=b"abc123",
  79. ):
  80. test = Test(
  81. name="name",
  82. cluster={"byod": {"post_build_script": "foo.sh"}},
  83. )
  84. build_anyscale_custom_byod_image(test)
  85. assert "docker build --build-arg BASE_IMAGE=029272617770.dkr.ecr.us-west-2."
  86. "amazonaws.com/anyscale/ray:abc123-py37 -t 029272617770.dkr.ecr.us-west-2."
  87. "amazonaws.com/anyscale/ray:abc123-py37-c3fc5fc6d84cea4d7ab885c6cdc966542e"
  88. "f59e4c679b8c970f2f77b956bfd8fb" in " ".join(cmds[0])
  89. def test_build_anyscale_base_byod_images() -> None:
  90. images = []
  91. def _mock_validate_and_push(image: str) -> None:
  92. images.append(image)
  93. def _mock_image_exist(image: str) -> bool:
  94. return "rayproject/ray" in image
  95. with patch(
  96. "ray_release.byod.build._download_dataplane_build_file", return_value=None
  97. ), patch(
  98. "os.environ",
  99. {"BUILDKITE_COMMIT": "abc123", "BUILDKITE_BRANCH": "master"},
  100. ), patch(
  101. "subprocess.check_call", return_value=None
  102. ), patch(
  103. "ray_release.byod.build._image_exist", side_effect=_mock_image_exist
  104. ), patch(
  105. "ray_release.byod.build._validate_and_push", side_effect=_mock_validate_and_push
  106. ):
  107. tests = [
  108. Test(name="aws", env="aws", cluster={"byod": {}}),
  109. Test(name="aws", env="aws", cluster={"byod": {"type": "gpu"}}),
  110. Test(
  111. # This is a duplicate of the default.
  112. name="aws",
  113. env="aws",
  114. python="3.9",
  115. cluster={"byod": {"type": "cpu"}},
  116. ),
  117. Test(name="aws", env="aws", cluster={"byod": {"type": "cu121"}}),
  118. Test(
  119. name="aws", env="aws", python="3.9", cluster={"byod": {"type": "cu116"}}
  120. ),
  121. Test(
  122. name="aws",
  123. env="aws",
  124. python="3.11",
  125. cluster={"byod": {"type": "cu118"}},
  126. ),
  127. Test(name="gce", env="gce", cluster={"byod": {}}),
  128. ]
  129. build_anyscale_base_byod_images(tests)
  130. global_config = get_global_config()
  131. aws_cr = global_config["byod_aws_cr"]
  132. gcp_cr = global_config["byod_gcp_cr"]
  133. assert images == [
  134. f"{aws_cr}/anyscale/ray:abc123-py39-cpu",
  135. f"{aws_cr}/anyscale/ray-ml:abc123-py39-gpu",
  136. f"{aws_cr}/anyscale/ray:abc123-py39-cu121",
  137. f"{aws_cr}/anyscale/ray:abc123-py39-cu116",
  138. f"{aws_cr}/anyscale/ray:abc123-py311-cu118",
  139. f"{gcp_cr}/anyscale/ray:abc123-py39-cpu",
  140. ]
  141. if __name__ == "__main__":
  142. sys.exit(pytest.main(["-v", __file__]))