build_pipeline.py 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. import json
  2. import os
  3. import shutil
  4. import sys
  5. from typing import Tuple
  6. from pathlib import Path
  7. import click
  8. from ray_release.buildkite.filter import filter_tests, group_tests
  9. from ray_release.buildkite.settings import get_pipeline_settings
  10. from ray_release.buildkite.step import get_step_for_test_group
  11. from ray_release.byod.build import (
  12. build_anyscale_base_byod_images,
  13. build_anyscale_custom_byod_image,
  14. )
  15. from ray_release.config import read_and_validate_release_test_collection
  16. from ray_release.configs.global_config import init_global_config
  17. from ray_release.exception import ReleaseTestCLIError, ReleaseTestConfigError
  18. from ray_release.logger import logger
  19. from ray_release.wheels import get_buildkite_repo_branch
  20. PIPELINE_ARTIFACT_PATH = "/tmp/pipeline_artifacts"
  21. @click.command()
  22. @click.option(
  23. "--test-collection-file",
  24. type=str,
  25. multiple=True,
  26. help="Test collection file, relative path to ray repo.",
  27. )
  28. @click.option(
  29. "--run-jailed-tests",
  30. is_flag=True,
  31. show_default=True,
  32. default=False,
  33. help=("Will run jailed tests."),
  34. )
  35. @click.option(
  36. "--run-unstable-tests",
  37. is_flag=True,
  38. show_default=True,
  39. default=False,
  40. help=("Will run unstable tests."),
  41. )
  42. @click.option(
  43. "--global-config",
  44. default="oss_config.yaml",
  45. type=click.Choice(
  46. [x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
  47. ),
  48. help="Global config to use for test execution.",
  49. )
  50. @click.option(
  51. "--run-per-test",
  52. default=1,
  53. type=int,
  54. help=("The number of time we run test on the same commit"),
  55. )
  56. def main(
  57. test_collection_file: Tuple[str],
  58. run_jailed_tests: bool = False,
  59. run_unstable_tests: bool = False,
  60. global_config: str = "oss_config.yaml",
  61. run_per_test: int = 1,
  62. ):
  63. global_config_file = os.path.join(
  64. os.path.dirname(__file__), "..", "configs", global_config
  65. )
  66. init_global_config(global_config_file)
  67. settings = get_pipeline_settings()
  68. tmpdir = None
  69. env = {}
  70. frequency = settings["frequency"]
  71. prefer_smoke_tests = settings["prefer_smoke_tests"]
  72. test_attr_regex_filters = settings["test_attr_regex_filters"]
  73. priority = settings["priority"]
  74. logger.info(
  75. f"Found the following buildkite pipeline settings:\n\n"
  76. f" frequency = {settings['frequency']}\n"
  77. f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n"
  78. f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
  79. f" ray_test_repo = {settings['ray_test_repo']}\n"
  80. f" ray_test_branch = {settings['ray_test_branch']}\n"
  81. f" priority = {settings['priority']}\n"
  82. f" no_concurrency_limit = {settings['no_concurrency_limit']}\n"
  83. )
  84. try:
  85. test_collection = read_and_validate_release_test_collection(
  86. test_collection_file or ["release/release_tests.yaml"]
  87. )
  88. except ReleaseTestConfigError as e:
  89. raise ReleaseTestConfigError(
  90. "Cannot load test yaml file.\nHINT: If you're kicking off tests for a "
  91. "specific commit on Buildkite to test Ray wheels, after clicking "
  92. "'New build', leave the commit at HEAD, and only specify the commit "
  93. "in the dialog that asks for the Ray wheels."
  94. ) from e
  95. if tmpdir:
  96. shutil.rmtree(tmpdir, ignore_errors=True)
  97. filtered_tests = filter_tests(
  98. test_collection,
  99. frequency=frequency,
  100. test_attr_regex_filters=test_attr_regex_filters,
  101. prefer_smoke_tests=prefer_smoke_tests,
  102. run_jailed_tests=run_jailed_tests,
  103. run_unstable_tests=run_unstable_tests,
  104. )
  105. logger.info(f"Found {len(filtered_tests)} tests to run.")
  106. if len(filtered_tests) == 0:
  107. raise ReleaseTestCLIError(
  108. "Empty test collection. The selected frequency or filter did "
  109. "not return any tests to run. Adjust your filters."
  110. )
  111. tests = [test for test, _ in filtered_tests]
  112. logger.info("Build anyscale base BYOD images")
  113. build_anyscale_base_byod_images(tests)
  114. logger.info("Build anyscale custom BYOD images")
  115. for test in tests:
  116. build_anyscale_custom_byod_image(test)
  117. grouped_tests = group_tests(filtered_tests)
  118. group_str = ""
  119. for group, tests in grouped_tests.items():
  120. group_str += f"\n{group}:\n"
  121. for test, smoke in tests:
  122. group_str += f" {test['name']}"
  123. if smoke:
  124. group_str += " [smoke test]"
  125. group_str += "\n"
  126. logger.info(f"Tests to run:\n{group_str}")
  127. no_concurrency_limit = settings["no_concurrency_limit"]
  128. if no_concurrency_limit:
  129. logger.warning("Concurrency is not limited for this run!")
  130. _, buildkite_branch = get_buildkite_repo_branch()
  131. if os.environ.get("REPORT_TO_RAY_TEST_DB", False):
  132. env["REPORT_TO_RAY_TEST_DB"] = "1"
  133. steps = get_step_for_test_group(
  134. grouped_tests,
  135. minimum_run_per_test=run_per_test,
  136. test_collection_file=test_collection_file,
  137. env=env,
  138. priority=priority.value,
  139. global_config=global_config,
  140. is_concurrency_limit=not no_concurrency_limit,
  141. )
  142. if "BUILDKITE" in os.environ:
  143. if os.path.exists(PIPELINE_ARTIFACT_PATH):
  144. shutil.rmtree(PIPELINE_ARTIFACT_PATH)
  145. os.makedirs(PIPELINE_ARTIFACT_PATH, exist_ok=True, mode=0o755)
  146. with open(os.path.join(PIPELINE_ARTIFACT_PATH, "pipeline.json"), "wt") as fp:
  147. json.dump(steps, fp)
  148. settings["frequency"] = settings["frequency"].value
  149. settings["priority"] = settings["priority"].value
  150. with open(os.path.join(PIPELINE_ARTIFACT_PATH, "settings.json"), "wt") as fp:
  151. json.dump(settings, fp)
  152. steps_str = json.dumps(steps)
  153. print(steps_str)
  154. if __name__ == "__main__":
  155. sys.exit(main())