test_env_utils.py 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. from __future__ import annotations
  2. import hashlib
  3. import os
  4. import subprocess
  5. import pytest
  6. from sweagent.environment.utils import (
  7. InvalidGithubURL,
  8. format_trajectory_markdown,
  9. get_associated_commit_urls,
  10. get_instances,
  11. is_github_issue_url,
  12. is_github_repo_url,
  13. parse_gh_issue_url,
  14. parse_gh_repo_url,
  15. remove_triple_backticks,
  16. )
  17. from sweagent.utils.config import keys_config
  18. _TOKEN = {"token": keys_config["GITHUB_TOKEN"]}
  19. def test_format_trajectory_markdown(test_trajectory):
  20. formatted = format_trajectory_markdown(test_trajectory["trajectory"])
  21. assert formatted.startswith("<details>")
  22. assert formatted.endswith("</details>")
  23. def test_remove_triple_backticks():
  24. assert remove_triple_backticks("```") == ""
  25. def test_is_github_repo_url():
  26. assert is_github_repo_url("https://github.com/princeton-nlp/SWE-agent")
  27. assert is_github_repo_url("https://github.com/princeton-nlp/SWE-agent/anything")
  28. assert is_github_repo_url("github.com/princeton-nlp/SWE-agent/anything")
  29. assert not is_github_repo_url("")
  30. assert not is_github_repo_url("/path/to/file")
  31. def test_parse_gh_repo_url():
  32. assert parse_gh_repo_url("https://github.com/princeton-nlp/SWE-agent") == ("princeton-nlp", "SWE-agent")
  33. assert parse_gh_repo_url("github.com/princeton-nlp/SWE-agent") == ("princeton-nlp", "SWE-agent")
  34. assert parse_gh_repo_url("github.com/princeton-nlp/SWE-agent/asdfjsdfg") == ("princeton-nlp", "SWE-agent")
  35. assert parse_gh_repo_url("git@github.com/princeton-nlp/SWE-agent/asdfjsdfg") == ("princeton-nlp", "SWE-agent")
  36. def test_parse_gh_repo_url_fails():
  37. with pytest.raises(InvalidGithubURL):
  38. parse_gh_repo_url("adfkj;lasdfl;kj")
  39. with pytest.raises(InvalidGithubURL):
  40. parse_gh_repo_url("github.com/")
  41. with pytest.raises(InvalidGithubURL):
  42. parse_gh_repo_url("github.com//a/")
  43. def test_parse_gh_issue_url():
  44. url = "https://github.com/princeton-nlp/SWE-agent/issues/43"
  45. owner, repo, no = parse_gh_issue_url(url)
  46. assert owner == "princeton-nlp"
  47. assert repo == "SWE-agent"
  48. assert no == "43"
  49. def test_parse_gh_issue_url_fails():
  50. with pytest.raises(InvalidGithubURL):
  51. parse_gh_issue_url("https://github.com/a/b")
  52. with pytest.raises(InvalidGithubURL):
  53. parse_gh_issue_url("https://github.com/a/b////")
  54. def test_is_from_github_url():
  55. assert not is_github_issue_url("")
  56. assert is_github_issue_url("https://github.com/princeton-nlp/SWE-agent/issues/43")
  57. def test_get_associated_commit_urls():
  58. assoc = get_associated_commit_urls(
  59. org="princeton-nlp",
  60. repo="SWE-agent",
  61. issue_number="41",
  62. token=os.environ.get("GITHUB_TOKEN", ""),
  63. )
  64. assert len(assoc) > 0
  65. def test_get_instance_gh_issue():
  66. instance = get_instances("https://github.com/swe-agent/test-repo/issues/1", **_TOKEN)[0]
  67. compare_with = {
  68. "repo": "swe-agent/test-repo",
  69. "instance_id": "swe-agent__test-repo-i1",
  70. "repo_type": "github",
  71. }
  72. for key in compare_with:
  73. assert instance[key] == compare_with[key]
  74. assert "SyntaxError" in instance["problem_statement"]
  75. assert len(instance["base_commit"]) > 10
  76. assert instance["version"]
  77. def clone_repo(tmp_path, repo_url):
  78. cmd = [
  79. "git",
  80. "clone",
  81. repo_url,
  82. ]
  83. subprocess.run(cmd, check=True, cwd=tmp_path)
  84. def test_get_instance_gh_issue_local_repo(tmp_path):
  85. clone_repo(tmp_path, "https://github.com/swe-agent/test-repo/")
  86. instance = get_instances(
  87. file_path="https://github.com/swe-agent/test-repo/issues/1",
  88. repo_path=str(tmp_path / "test-repo"),
  89. **_TOKEN,
  90. )[0]
  91. compare_with = {
  92. "repo": str(tmp_path.resolve() / "test-repo"),
  93. "repo_type": "local",
  94. "instance_id": "swe-agent__test-repo-i1",
  95. }
  96. for key in compare_with:
  97. assert instance[key] == compare_with[key]
  98. assert "SyntaxError" in instance["problem_statement"]
  99. assert len(instance["base_commit"]) > 10
  100. assert instance["version"]
  101. def test_get_instance_local_issue_local_repo(tmp_path):
  102. clone_repo(tmp_path, "https://github.com/swe-agent/test-repo/")
  103. issue_path = tmp_path / "issue.txt"
  104. issue_path.write_text("asdf")
  105. instance = get_instances(
  106. file_path=str(issue_path),
  107. repo_path=str(tmp_path / "test-repo"),
  108. )[0]
  109. compare_with = {
  110. "repo": str(tmp_path.resolve() / "test-repo"),
  111. "repo_type": "local",
  112. "instance_id": hashlib.sha256(b"asdf").hexdigest()[:6],
  113. "problem_statement": "asdf",
  114. }
  115. for key in compare_with:
  116. assert instance[key] == compare_with[key]
  117. assert len(instance["base_commit"]) > 10
  118. assert instance["version"]
  119. def test_get_instance_gh_issue_gh_repo(tmp_path):
  120. instance = get_instances(
  121. file_path="https://github.com/swe-agent/test-repo/issues/1",
  122. repo_path="https://github.com/princeton-nlp/SWE-agent",
  123. **_TOKEN,
  124. )[0]
  125. compare_with = {
  126. "repo": "princeton-nlp/SWE-agent",
  127. "repo_type": "github",
  128. "instance_id": "swe-agent__test-repo-i1",
  129. }
  130. for key in compare_with:
  131. assert instance[key] == compare_with[key]
  132. assert "SyntaxError" in instance["problem_statement"]
  133. assert len(instance["base_commit"]) > 10
  134. assert instance["version"]
  135. def test_get_instance_text_issue_gh_repo(tmp_path):
  136. instance = get_instances(
  137. file_path="text://this is a test",
  138. repo_path="https://github.com/princeton-nlp/SWE-agent",
  139. **_TOKEN,
  140. )[0]
  141. compare_with = {
  142. "repo": "princeton-nlp/SWE-agent",
  143. "repo_type": "github",
  144. "problem_statement": "this is a test",
  145. }
  146. for key in compare_with:
  147. assert instance[key] == compare_with[key]
  148. assert len(instance["base_commit"]) > 10
  149. assert instance["version"]
  150. def test_load_instances(test_data_path, caplog):
  151. test_data_sources = test_data_path / "data_sources"
  152. examples = [example for example in test_data_sources.iterdir() if example.is_file()]
  153. for example in examples:
  154. get_instances(file_path=str(example), **_TOKEN)
  155. @pytest.mark.ctf
  156. def test_load_ctf_instances(test_data_path, caplog):
  157. test_data_sources = test_data_path / "data_sources" / "ctf"
  158. examples = list(test_data_sources.glob("**/challenge.json"))
  159. for example in examples:
  160. get_instances(file_path=str(example), repo_path=str(example.parent))