test_state_machine.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. import sys
  2. from typing import List, Optional
  3. import pytest
  4. from ray_release.test import (
  5. Test,
  6. TestResult,
  7. TestState,
  8. )
  9. from ray_release.result import (
  10. Result,
  11. ResultStatus,
  12. )
  13. from ray_release.test_automation.release_state_machine import ReleaseTestStateMachine
  14. from ray_release.test_automation.ci_state_machine import (
  15. CITestStateMachine,
  16. CONTINUOUS_FAILURE_TO_FLAKY,
  17. CONTINUOUS_PASSING_TO_PASSING,
  18. FAILING_TO_FLAKY_MESSAGE,
  19. JAILED_TAG,
  20. JAILED_MESSAGE,
  21. )
  22. from ray_release.test_automation.state_machine import (
  23. TestStateMachine,
  24. WEEKLY_RELEASE_BLOCKER_TAG,
  25. NO_TEAM,
  26. )
  27. class MockLabel:
  28. def __init__(self, name: str):
  29. self.name = name
  30. class MockIssue:
  31. def __init__(
  32. self,
  33. number: int,
  34. title: str,
  35. state: str = "open",
  36. labels: Optional[List[MockLabel]] = None,
  37. ):
  38. self.number = number
  39. self.title = title
  40. self.state = state
  41. self.labels = labels or []
  42. self.comments = []
  43. def edit(
  44. self, state: str = None, labels: List[MockLabel] = None, title: str = None
  45. ):
  46. if state:
  47. self.state = state
  48. if labels:
  49. self.labels = labels
  50. if title:
  51. self.title = title
  52. if state:
  53. self.state = state
  54. def create_comment(self, comment: str):
  55. self.comments.append(comment)
  56. def get_labels(self):
  57. return self.labels
  58. class MockIssueDB:
  59. issue_id = 1
  60. issue_db = {}
  61. class MockRepo:
  62. def create_issue(self, labels: List[str], title: str, *args, **kwargs):
  63. label_objs = [MockLabel(label) for label in labels]
  64. issue = MockIssue(MockIssueDB.issue_id, title=title, labels=label_objs)
  65. MockIssueDB.issue_db[MockIssueDB.issue_id] = issue
  66. MockIssueDB.issue_id += 1
  67. return issue
  68. def get_issue(self, number: int):
  69. return MockIssueDB.issue_db[number]
  70. def get_issues(self, state: str, labels: List[MockLabel]) -> List[MockIssue]:
  71. issues = []
  72. for issue in MockIssueDB.issue_db.values():
  73. if issue.state != state:
  74. continue
  75. issue_labels = [label.name for label in issue.labels]
  76. if all(label.name in issue_labels for label in labels):
  77. issues.append(issue)
  78. return issues
  79. def get_label(self, name: str):
  80. return MockLabel(name)
  81. class MockBuildkiteBuild:
  82. def create_build(self, *args, **kwargs):
  83. return {
  84. "number": 1,
  85. "jobs": [{"id": "1"}],
  86. }
  87. def list_all_for_pipeline(self, *args, **kwargs):
  88. return []
  89. class MockBuildkiteJob:
  90. def unblock_job(self, *args, **kwargs):
  91. return {}
  92. class MockBuildkite:
  93. def builds(self):
  94. return MockBuildkiteBuild()
  95. def jobs(self):
  96. return MockBuildkiteJob()
  97. TestStateMachine.ray_repo = MockRepo()
  98. TestStateMachine.ray_buildkite = MockBuildkite()
  99. def test_ci_empty_results():
  100. test = Test(name="w00t", team="ci", state=TestState.FLAKY)
  101. test.test_results = []
  102. CITestStateMachine(test).move()
  103. # do not change the state
  104. assert test.get_state() == TestState.FLAKY
  105. def test_ci_move_from_passing_to_flaky():
  106. """
  107. Test the entire lifecycle of a CI test when it moves from passing to flaky.
  108. """
  109. test = Test(name="w00t", team="ci")
  110. # start from passing
  111. assert test.get_state() == TestState.PASSING
  112. # passing to flaky
  113. test.test_results = [
  114. TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
  115. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  116. ] * 10
  117. CITestStateMachine(test).move()
  118. assert test.get_state() == TestState.FLAKY
  119. issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
  120. assert issue.state == "open"
  121. assert issue.title == "CI test w00t is flaky"
  122. # flaky to jail
  123. issue.edit(labels=[MockLabel(JAILED_TAG)])
  124. CITestStateMachine(test).move()
  125. assert test.get_state() == TestState.JAILED
  126. assert issue.comments[-1] == JAILED_MESSAGE
  127. def test_ci_move_from_passing_to_failing_to_flaky():
  128. """
  129. Test the entire lifecycle of a CI test when it moves from passing to failing.
  130. Check that the conditions are met for each state transition. Also check that
  131. gihub issues are created and closed correctly.
  132. """
  133. test = Test(name="test", team="ci")
  134. # start from passing
  135. assert test.get_state() == TestState.PASSING
  136. # passing to failing
  137. test.test_results = [
  138. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  139. ]
  140. CITestStateMachine(test).move()
  141. assert test.get_state() == TestState.FAILING
  142. # failing to consistently failing
  143. test.test_results.extend(
  144. [
  145. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  146. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  147. ]
  148. )
  149. CITestStateMachine(test).move()
  150. assert test.get_state() == TestState.CONSITENTLY_FAILING
  151. issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
  152. assert issue.state == "open"
  153. assert "ci-test" in [label.name for label in issue.labels]
  154. # move from consistently failing to flaky
  155. test.test_results.extend(
  156. [TestResult.from_result(Result(status=ResultStatus.ERROR.value))]
  157. * CONTINUOUS_FAILURE_TO_FLAKY
  158. )
  159. CITestStateMachine(test).move()
  160. assert test.get_state() == TestState.FLAKY
  161. assert issue.comments[-1] == FAILING_TO_FLAKY_MESSAGE
  162. # go back to passing
  163. test.test_results = [
  164. TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
  165. ] * CONTINUOUS_PASSING_TO_PASSING
  166. CITestStateMachine(test).move()
  167. assert test.get_state() == TestState.PASSING
  168. assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
  169. assert issue.state == "closed"
  170. # go back to failing and reuse the github issue
  171. test.test_results = 3 * [
  172. TestResult.from_result(Result(status=ResultStatus.ERROR.value))
  173. ]
  174. CITestStateMachine(test).move()
  175. assert test.get_state() == TestState.CONSITENTLY_FAILING
  176. assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
  177. assert issue.state == "open"
  178. def test_release_move_from_passing_to_failing():
  179. test = Test(name="test", team="ci")
  180. # Test original state
  181. test.test_results = [
  182. TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
  183. ]
  184. assert test.get_state() == TestState.PASSING
  185. # Test moving from passing to failing
  186. test.test_results.insert(
  187. 0,
  188. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  189. )
  190. sm = ReleaseTestStateMachine(test)
  191. sm.move()
  192. assert test.get_state() == TestState.FAILING
  193. assert test[Test.KEY_BISECT_BUILD_NUMBER] == 1
  194. # Test moving from failing to consistently failing
  195. test.test_results.insert(
  196. 0,
  197. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  198. )
  199. sm = ReleaseTestStateMachine(test)
  200. sm.move()
  201. assert test.get_state() == TestState.CONSITENTLY_FAILING
  202. assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
  203. def test_release_move_from_failing_to_consisently_failing():
  204. test = Test(name="test", team="ci", stable=False)
  205. test[Test.KEY_BISECT_BUILD_NUMBER] = 1
  206. test.test_results = [
  207. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  208. ]
  209. sm = ReleaseTestStateMachine(test)
  210. sm.move()
  211. assert test.get_state() == TestState.FAILING
  212. test[Test.KEY_BISECT_BLAMED_COMMIT] = "1234567890"
  213. sm = ReleaseTestStateMachine(test)
  214. sm.move()
  215. sm.comment_blamed_commit_on_github_issue()
  216. issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
  217. assert test.get_state() == TestState.CONSITENTLY_FAILING
  218. assert "Blamed commit: 1234567890" in issue.comments[0]
  219. labels = [label.name for label in issue.get_labels()]
  220. assert "ci" in labels
  221. assert "unstable-release-test" in labels
  222. def test_release_move_from_failing_to_passing():
  223. test = Test(name="test", team="ci")
  224. test.test_results = [
  225. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  226. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  227. ]
  228. sm = ReleaseTestStateMachine(test)
  229. sm.move()
  230. assert test.get_state() == TestState.CONSITENTLY_FAILING
  231. assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
  232. test.test_results.insert(
  233. 0,
  234. TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
  235. )
  236. sm = ReleaseTestStateMachine(test)
  237. sm.move()
  238. assert test.get_state() == TestState.PASSING
  239. assert test.get(Test.KEY_BISECT_BUILD_NUMBER) is None
  240. assert test.get(Test.KEY_BISECT_BLAMED_COMMIT) is None
  241. def test_release_move_from_failing_to_jailed():
  242. test = Test(name="test", team="ci")
  243. test.test_results = [
  244. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  245. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  246. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  247. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  248. ]
  249. sm = ReleaseTestStateMachine(test)
  250. sm.move()
  251. assert test.get_state() == TestState.CONSITENTLY_FAILING
  252. test.test_results.insert(
  253. 0,
  254. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  255. )
  256. sm = ReleaseTestStateMachine(test)
  257. sm.move()
  258. assert test.get_state() == TestState.JAILED
  259. # Test moving from jailed to jailed
  260. issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
  261. issue.edit(state="closed")
  262. test.test_results.insert(
  263. 0,
  264. TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
  265. )
  266. sm = ReleaseTestStateMachine(test)
  267. sm.move()
  268. assert test.get_state() == TestState.JAILED
  269. assert issue.state == "open"
  270. # Test moving from jailed to passing
  271. test.test_results.insert(
  272. 0,
  273. TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
  274. )
  275. sm = ReleaseTestStateMachine(test)
  276. sm.move()
  277. assert test.get_state() == TestState.PASSING
  278. assert issue.state == "closed"
  279. def test_get_release_blockers() -> None:
  280. MockIssueDB.issue_id = 1
  281. MockIssueDB.issue_db = {}
  282. TestStateMachine.ray_repo.create_issue(labels=["non-blocker"], title="non-blocker")
  283. TestStateMachine.ray_repo.create_issue(
  284. labels=[WEEKLY_RELEASE_BLOCKER_TAG], title="blocker"
  285. )
  286. issues = TestStateMachine.get_release_blockers()
  287. assert len(issues) == 1
  288. assert issues[0].title == "blocker"
  289. def test_get_issue_owner() -> None:
  290. issue = TestStateMachine.ray_repo.create_issue(labels=["core"], title="hi")
  291. assert TestStateMachine.get_issue_owner(issue) == "core"
  292. issue = TestStateMachine.ray_repo.create_issue(labels=["w00t"], title="bye")
  293. assert TestStateMachine.get_issue_owner(issue) == NO_TEAM
  294. if __name__ == "__main__":
  295. sys.exit(pytest.main(["-v", __file__]))