run_release_test.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. import os
  2. import sys
  3. from typing import Optional, Tuple
  4. from pathlib import Path
  5. import click
  6. from ray_release.aws import maybe_fetch_api_token
  7. from ray_release.config import (
  8. as_smoke_test,
  9. find_test,
  10. read_and_validate_release_test_collection,
  11. )
  12. from ray_release.configs.global_config import init_global_config
  13. from ray_release.env import DEFAULT_ENVIRONMENT, load_environment, populate_os_env
  14. from ray_release.exception import ReleaseTestCLIError, ReleaseTestError
  15. from ray_release.glue import run_release_test
  16. from ray_release.logger import logger
  17. from ray_release.reporter.artifacts import ArtifactsReporter
  18. from ray_release.reporter.db import DBReporter
  19. from ray_release.reporter.ray_test_db import RayTestDBReporter
  20. from ray_release.reporter.log import LogReporter
  21. from ray_release.result import Result
  22. from ray_release.anyscale_util import LAST_LOGS_LENGTH
  23. @click.command()
  24. @click.argument("test_name", required=True, type=str)
  25. @click.option(
  26. "--test-collection-file",
  27. multiple=True,
  28. type=str,
  29. help="Test collection file, relative path to ray repo.",
  30. )
  31. @click.option(
  32. "--smoke-test",
  33. default=False,
  34. type=bool,
  35. is_flag=True,
  36. help="Finish quickly for testing",
  37. )
  38. @click.option(
  39. "--report",
  40. default=False,
  41. type=bool,
  42. is_flag=True,
  43. help="Report results to database",
  44. )
  45. @click.option(
  46. "--cluster-id",
  47. default=None,
  48. type=str,
  49. help="Cluster ID of existing cluster to be re-used.",
  50. )
  51. @click.option(
  52. "--cluster-env-id",
  53. default=None,
  54. type=str,
  55. help="Cluster env ID of existing cluster env to be re-used.",
  56. )
  57. @click.option(
  58. "--env",
  59. default=None,
  60. # Get the names without suffixes of all files in "../environments"
  61. type=click.Choice(
  62. [x.stem for x in (Path(__file__).parent.parent / "environments").glob("*.env")]
  63. ),
  64. help="Environment to use. Will overwrite environment used in test config.",
  65. )
  66. @click.option(
  67. "--global-config",
  68. default="oss_config.yaml",
  69. type=click.Choice(
  70. [x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
  71. ),
  72. help="Global config to use for test execution.",
  73. )
  74. @click.option(
  75. "--no-terminate",
  76. default=False,
  77. type=bool,
  78. is_flag=True,
  79. help=(
  80. "Do not terminate cluster after test. "
  81. "Will switch `anyscale_job` run type to `job` (Ray Job)."
  82. ),
  83. )
  84. @click.option(
  85. "--test-definition-root",
  86. default=None,
  87. type=str,
  88. help="Root of the test definition files. Default is the root of the repo.",
  89. )
  90. @click.option(
  91. "--log-streaming-limit",
  92. default=LAST_LOGS_LENGTH,
  93. type=int,
  94. help="Limit of log streaming in number of lines. Set to -1 to stream all logs.",
  95. )
  96. def main(
  97. test_name: str,
  98. test_collection_file: Tuple[str],
  99. smoke_test: bool = False,
  100. report: bool = False,
  101. cluster_id: Optional[str] = None,
  102. cluster_env_id: Optional[str] = None,
  103. env: Optional[str] = None,
  104. global_config: str = "oss_config.yaml",
  105. no_terminate: bool = False,
  106. test_definition_root: Optional[str] = None,
  107. log_streaming_limit: int = LAST_LOGS_LENGTH,
  108. ):
  109. global_config_file = os.path.join(
  110. os.path.dirname(__file__), "..", "configs", global_config
  111. )
  112. init_global_config(global_config_file)
  113. test_collection = read_and_validate_release_test_collection(
  114. test_collection_file or ["release/release_tests.yaml"],
  115. test_definition_root,
  116. )
  117. test = find_test(test_collection, test_name)
  118. if not test:
  119. raise ReleaseTestCLIError(
  120. f"Test `{test_name}` not found in collection file: "
  121. f"{test_collection_file}"
  122. )
  123. if smoke_test:
  124. test = as_smoke_test(test)
  125. env_to_use = env or test.get("env", DEFAULT_ENVIRONMENT)
  126. env_dict = load_environment(env_to_use)
  127. populate_os_env(env_dict)
  128. anyscale_project = os.environ.get("ANYSCALE_PROJECT", None)
  129. if not anyscale_project:
  130. raise ReleaseTestCLIError(
  131. "You have to set the ANYSCALE_PROJECT environment variable!"
  132. )
  133. maybe_fetch_api_token()
  134. result = Result()
  135. reporters = [LogReporter()]
  136. if "BUILDKITE" in os.environ:
  137. reporters.append(ArtifactsReporter())
  138. if report:
  139. reporters.append(DBReporter())
  140. # TODO(can): this env var is used as a feature flag, in case we need to turn this
  141. # off quickly. We should remove this when the new db reporter is stable.
  142. if os.environ.get("REPORT_TO_RAY_TEST_DB", False):
  143. reporters.append(RayTestDBReporter())
  144. try:
  145. result = run_release_test(
  146. test,
  147. anyscale_project=anyscale_project,
  148. result=result,
  149. reporters=reporters,
  150. smoke_test=smoke_test,
  151. cluster_id=cluster_id,
  152. cluster_env_id=cluster_env_id,
  153. no_terminate=no_terminate,
  154. test_definition_root=test_definition_root,
  155. log_streaming_limit=log_streaming_limit,
  156. )
  157. return_code = result.return_code
  158. except ReleaseTestError as e:
  159. logger.exception(e)
  160. return_code = e.exit_code.value
  161. logger.info(
  162. f"Release test pipeline for test {test['name']} completed. "
  163. f"Returning with exit code = {return_code}"
  164. )
  165. sys.exit(return_code)
  166. if __name__ == "__main__":
  167. main()