test_run_script.py 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. import json
  2. import os
  3. import subprocess
  4. import sys
  5. import tempfile
  6. import pytest
  7. from ray_release.result import ExitCode
  8. @pytest.fixture
  9. def setup(tmpdir):
  10. state_file = os.path.join(tmpdir, "state.txt")
  11. test_script = os.path.join(
  12. os.path.dirname(__file__), "..", "..", "run_release_test.sh"
  13. )
  14. os.environ["NO_INSTALL"] = "1"
  15. os.environ["NO_CLONE"] = "1"
  16. os.environ["NO_ARTIFACTS"] = "1"
  17. os.environ[
  18. "RAY_TEST_SCRIPT"
  19. ] = "python ray_release/tests/_test_run_release_test_sh.py"
  20. os.environ["OVERRIDE_SLEEP_TIME"] = "0"
  21. os.environ["MAX_RETRIES"] = "3"
  22. yield state_file, test_script
  23. def _read_state(state_file):
  24. with open(state_file, "rt") as f:
  25. return int(f.read())
  26. def _run_script(test_script, state_file, *exits):
  27. assert len(exits) == 3
  28. if os.path.exists(state_file):
  29. os.unlink(state_file)
  30. try:
  31. return subprocess.check_call(
  32. f"{test_script} "
  33. f"{state_file} "
  34. f"{' '.join(str(e.value) for e in exits)}",
  35. shell=True,
  36. )
  37. except subprocess.CalledProcessError as e:
  38. return e.returncode
  39. def test_repeat(setup):
  40. state_file, test_script = setup
  41. assert (
  42. _run_script(
  43. test_script,
  44. state_file,
  45. ExitCode.SUCCESS,
  46. ExitCode.SUCCESS,
  47. ExitCode.SUCCESS,
  48. )
  49. == ExitCode.SUCCESS.value
  50. )
  51. assert _read_state(state_file) == 1
  52. assert (
  53. _run_script(
  54. test_script,
  55. state_file,
  56. ExitCode.RAY_WHEELS_TIMEOUT,
  57. ExitCode.SUCCESS,
  58. ExitCode.SUCCESS,
  59. )
  60. == ExitCode.SUCCESS.value
  61. )
  62. assert _read_state(state_file) == 2
  63. assert (
  64. _run_script(
  65. test_script,
  66. state_file,
  67. ExitCode.RAY_WHEELS_TIMEOUT,
  68. ExitCode.CLUSTER_ENV_BUILD_TIMEOUT,
  69. ExitCode.SUCCESS,
  70. )
  71. == ExitCode.SUCCESS.value
  72. )
  73. assert _read_state(state_file) == 3
  74. assert (
  75. _run_script(
  76. test_script,
  77. state_file,
  78. ExitCode.CLUSTER_STARTUP_TIMEOUT,
  79. ExitCode.CLUSTER_WAIT_TIMEOUT,
  80. ExitCode.RAY_WHEELS_TIMEOUT,
  81. )
  82. == 79 # BUILDKITE_RETRY_CODE
  83. )
  84. assert _read_state(state_file) == 3
  85. assert (
  86. _run_script(
  87. test_script,
  88. state_file,
  89. ExitCode.RAY_WHEELS_TIMEOUT,
  90. ExitCode.COMMAND_ALERT,
  91. ExitCode.SUCCESS,
  92. )
  93. == 79 # BUILDKITE_RETRY_CODE
  94. )
  95. assert _read_state(state_file) == 2
  96. def test_parameters(setup):
  97. state_file, test_script = setup
  98. os.environ["RAY_TEST_SCRIPT"] = "python ray_release/tests/_test_catch_args.py"
  99. with tempfile.TemporaryDirectory() as tmpdir:
  100. argv_file = os.path.join(tmpdir, "argv.json")
  101. subprocess.check_call(
  102. f"{test_script} " f"{argv_file} " f"--smoke-test",
  103. shell=True,
  104. )
  105. with open(argv_file, "rt") as fp:
  106. data = json.load(fp)
  107. assert "--smoke-test" in data
  108. if __name__ == "__main__":
  109. import pytest
  110. sys.exit(pytest.main(["-v", __file__]))