test_run_script.py 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136
  1. import json
  2. import os
  3. import subprocess
  4. import sys
  5. import tempfile
  6. import pytest
  7. from ray_release.result import ExitCode
  8. @pytest.fixture
  9. def setup(tmpdir):
  10. state_file = os.path.join(tmpdir, "state.txt")
  11. test_script = os.path.join(
  12. os.path.dirname(__file__), "..", "..", "run_release_test.sh"
  13. )
  14. os.environ["NO_INSTALL"] = "1"
  15. os.environ["NO_CLONE"] = "1"
  16. os.environ["NO_ARTIFACTS"] = "1"
  17. os.environ["RAY_TEST_SCRIPT"] = "ray_release/tests/_test_run_release_test_sh.py"
  18. os.environ["OVERRIDE_SLEEP_TIME"] = "0"
  19. os.environ["MAX_RETRIES"] = "3"
  20. yield state_file, test_script
  21. def _read_state(state_file):
  22. with open(state_file, "rt") as f:
  23. return int(f.read())
  24. def _run_script(test_script, state_file, *exits):
  25. assert len(exits) == 3
  26. if os.path.exists(state_file):
  27. os.unlink(state_file)
  28. try:
  29. return subprocess.check_call(
  30. f"{test_script} "
  31. f"{state_file} "
  32. f"{' '.join(str(e.value) for e in exits)}",
  33. shell=True,
  34. )
  35. except subprocess.CalledProcessError as e:
  36. return e.returncode
  37. def test_repeat(setup):
  38. state_file, test_script = setup
  39. assert (
  40. _run_script(
  41. test_script,
  42. state_file,
  43. ExitCode.SUCCESS,
  44. ExitCode.SUCCESS,
  45. ExitCode.SUCCESS,
  46. )
  47. == ExitCode.SUCCESS.value
  48. )
  49. assert _read_state(state_file) == 1
  50. assert (
  51. _run_script(
  52. test_script,
  53. state_file,
  54. ExitCode.RAY_WHEELS_TIMEOUT,
  55. ExitCode.SUCCESS,
  56. ExitCode.SUCCESS,
  57. )
  58. == ExitCode.SUCCESS.value
  59. )
  60. assert _read_state(state_file) == 2
  61. assert (
  62. _run_script(
  63. test_script,
  64. state_file,
  65. ExitCode.RAY_WHEELS_TIMEOUT,
  66. ExitCode.CLUSTER_ENV_BUILD_TIMEOUT,
  67. ExitCode.SUCCESS,
  68. )
  69. == ExitCode.SUCCESS.value
  70. )
  71. assert _read_state(state_file) == 3
  72. assert (
  73. _run_script(
  74. test_script,
  75. state_file,
  76. ExitCode.CLUSTER_STARTUP_TIMEOUT,
  77. ExitCode.CLUSTER_WAIT_TIMEOUT,
  78. ExitCode.RAY_WHEELS_TIMEOUT,
  79. )
  80. == 79 # BUILDKITE_RETRY_CODE
  81. )
  82. assert _read_state(state_file) == 3
  83. assert (
  84. _run_script(
  85. test_script,
  86. state_file,
  87. ExitCode.RAY_WHEELS_TIMEOUT,
  88. ExitCode.COMMAND_ALERT,
  89. ExitCode.SUCCESS,
  90. )
  91. == ExitCode.COMMAND_ALERT.value
  92. )
  93. assert _read_state(state_file) == 2
  94. def test_parameters(setup):
  95. state_file, test_script = setup
  96. os.environ["RAY_TEST_SCRIPT"] = "ray_release/tests/_test_catch_args.py"
  97. argv_file = tempfile.mktemp()
  98. subprocess.check_call(
  99. f"{test_script} " f"{argv_file} " f"--smoke-test",
  100. shell=True,
  101. )
  102. with open(argv_file, "rt") as fp:
  103. data = json.load(fp)
  104. os.unlink(argv_file)
  105. assert "--smoke-test" in data
  106. if __name__ == "__main__":
  107. import pytest
  108. sys.exit(pytest.main(["-v", __file__]))