123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353 |
- import sys
- from typing import List, Optional
- import pytest
- from ray_release.test import (
- Test,
- TestResult,
- TestState,
- )
- from ray_release.result import (
- Result,
- ResultStatus,
- )
- from ray_release.test_automation.release_state_machine import ReleaseTestStateMachine
- from ray_release.test_automation.ci_state_machine import (
- CITestStateMachine,
- CONTINUOUS_FAILURE_TO_FLAKY,
- CONTINUOUS_PASSING_TO_PASSING,
- FAILING_TO_FLAKY_MESSAGE,
- JAILED_TAG,
- JAILED_MESSAGE,
- )
- from ray_release.test_automation.state_machine import (
- TestStateMachine,
- WEEKLY_RELEASE_BLOCKER_TAG,
- NO_TEAM,
- )
- class MockLabel:
- def __init__(self, name: str):
- self.name = name
- class MockIssue:
- def __init__(
- self,
- number: int,
- title: str,
- state: str = "open",
- labels: Optional[List[MockLabel]] = None,
- ):
- self.number = number
- self.title = title
- self.state = state
- self.labels = labels or []
- self.comments = []
- def edit(
- self, state: str = None, labels: List[MockLabel] = None, title: str = None
- ):
- if state:
- self.state = state
- if labels:
- self.labels = labels
- if title:
- self.title = title
- if state:
- self.state = state
- def create_comment(self, comment: str):
- self.comments.append(comment)
- def get_labels(self):
- return self.labels
- class MockIssueDB:
- issue_id = 1
- issue_db = {}
- class MockRepo:
- def create_issue(self, labels: List[str], title: str, *args, **kwargs):
- label_objs = [MockLabel(label) for label in labels]
- issue = MockIssue(MockIssueDB.issue_id, title=title, labels=label_objs)
- MockIssueDB.issue_db[MockIssueDB.issue_id] = issue
- MockIssueDB.issue_id += 1
- return issue
- def get_issue(self, number: int):
- return MockIssueDB.issue_db[number]
- def get_issues(self, state: str, labels: List[MockLabel]) -> List[MockIssue]:
- issues = []
- for issue in MockIssueDB.issue_db.values():
- if issue.state != state:
- continue
- issue_labels = [label.name for label in issue.labels]
- if all(label.name in issue_labels for label in labels):
- issues.append(issue)
- return issues
- def get_label(self, name: str):
- return MockLabel(name)
- class MockBuildkiteBuild:
- def create_build(self, *args, **kwargs):
- return {
- "number": 1,
- "jobs": [{"id": "1"}],
- }
- def list_all_for_pipeline(self, *args, **kwargs):
- return []
- class MockBuildkiteJob:
- def unblock_job(self, *args, **kwargs):
- return {}
- class MockBuildkite:
- def builds(self):
- return MockBuildkiteBuild()
- def jobs(self):
- return MockBuildkiteJob()
- TestStateMachine.ray_repo = MockRepo()
- TestStateMachine.ray_buildkite = MockBuildkite()
- def test_ci_empty_results():
- test = Test(name="w00t", team="ci", state=TestState.FLAKY)
- test.test_results = []
- CITestStateMachine(test).move()
- # do not change the state
- assert test.get_state() == TestState.FLAKY
- def test_ci_move_from_passing_to_flaky():
- """
- Test the entire lifecycle of a CI test when it moves from passing to flaky.
- """
- test = Test(name="w00t", team="ci")
- # start from passing
- assert test.get_state() == TestState.PASSING
- # passing to flaky
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ] * 10
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.FLAKY
- issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
- assert issue.state == "open"
- assert issue.title == "CI test w00t is flaky"
- # flaky to jail
- issue.edit(labels=[MockLabel(JAILED_TAG)])
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.JAILED
- assert issue.comments[-1] == JAILED_MESSAGE
- def test_ci_move_from_passing_to_failing_to_flaky():
- """
- Test the entire lifecycle of a CI test when it moves from passing to failing.
- Check that the conditions are met for each state transition. Also check that
- gihub issues are created and closed correctly.
- """
- test = Test(name="test", team="ci")
- # start from passing
- assert test.get_state() == TestState.PASSING
- # passing to failing
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ]
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.FAILING
- # failing to consistently failing
- test.test_results.extend(
- [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ]
- )
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
- assert issue.state == "open"
- assert "ci-test" in [label.name for label in issue.labels]
- # move from consistently failing to flaky
- test.test_results.extend(
- [TestResult.from_result(Result(status=ResultStatus.ERROR.value))]
- * CONTINUOUS_FAILURE_TO_FLAKY
- )
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.FLAKY
- assert issue.comments[-1] == FAILING_TO_FLAKY_MESSAGE
- # go back to passing
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
- ] * CONTINUOUS_PASSING_TO_PASSING
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.PASSING
- assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
- assert issue.state == "closed"
- # go back to failing and reuse the github issue
- test.test_results = 3 * [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value))
- ]
- CITestStateMachine(test).move()
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
- assert issue.state == "open"
- def test_release_move_from_passing_to_failing():
- test = Test(name="test", team="ci")
- # Test original state
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
- ]
- assert test.get_state() == TestState.PASSING
- # Test moving from passing to failing
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.FAILING
- assert test[Test.KEY_BISECT_BUILD_NUMBER] == 1
- # Test moving from failing to consistently failing
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
- def test_release_move_from_failing_to_consisently_failing():
- test = Test(name="test", team="ci", stable=False)
- test[Test.KEY_BISECT_BUILD_NUMBER] = 1
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ]
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.FAILING
- test[Test.KEY_BISECT_BLAMED_COMMIT] = "1234567890"
- sm = ReleaseTestStateMachine(test)
- sm.move()
- sm.comment_blamed_commit_on_github_issue()
- issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- assert "Blamed commit: 1234567890" in issue.comments[0]
- labels = [label.name for label in issue.get_labels()]
- assert "ci" in labels
- assert "unstable-release-test" in labels
- def test_release_move_from_failing_to_passing():
- test = Test(name="test", team="ci")
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ]
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.PASSING
- assert test.get(Test.KEY_BISECT_BUILD_NUMBER) is None
- assert test.get(Test.KEY_BISECT_BLAMED_COMMIT) is None
- def test_release_move_from_failing_to_jailed():
- test = Test(name="test", team="ci")
- test.test_results = [
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- ]
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.CONSITENTLY_FAILING
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.JAILED
- # Test moving from jailed to jailed
- issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
- issue.edit(state="closed")
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.JAILED
- assert issue.state == "open"
- # Test moving from jailed to passing
- test.test_results.insert(
- 0,
- TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
- )
- sm = ReleaseTestStateMachine(test)
- sm.move()
- assert test.get_state() == TestState.PASSING
- assert issue.state == "closed"
- def test_get_release_blockers() -> None:
- MockIssueDB.issue_id = 1
- MockIssueDB.issue_db = {}
- TestStateMachine.ray_repo.create_issue(labels=["non-blocker"], title="non-blocker")
- TestStateMachine.ray_repo.create_issue(
- labels=[WEEKLY_RELEASE_BLOCKER_TAG], title="blocker"
- )
- issues = TestStateMachine.get_release_blockers()
- assert len(issues) == 1
- assert issues[0].title == "blocker"
- def test_get_issue_owner() -> None:
- issue = TestStateMachine.ray_repo.create_issue(labels=["core"], title="hi")
- assert TestStateMachine.get_issue_owner(issue) == "core"
- issue = TestStateMachine.ray_repo.create_issue(labels=["w00t"], title="bye")
- assert TestStateMachine.get_issue_owner(issue) == NO_TEAM
- if __name__ == "__main__":
- sys.exit(pytest.main(["-v", __file__]))
|