test_env_utils.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. from __future__ import annotations
  2. import hashlib
  3. import os
  4. import subprocess
  5. import pytest
  6. from sweagent.environment.utils import (
  7. InvalidGithubURL,
  8. format_trajectory_markdown,
  9. get_associated_commit_urls,
  10. get_instances,
  11. is_github_issue_url,
  12. is_github_repo_url,
  13. parse_gh_issue_url,
  14. parse_gh_repo_url,
  15. remove_triple_backticks,
  16. )
  17. _TOKEN = {"token": os.environ.get("GITHUB_TOKEN", "")}
  18. def test_format_trajectory_markdown(test_trajectory):
  19. formatted = format_trajectory_markdown(test_trajectory["trajectory"])
  20. assert formatted.startswith("<details>")
  21. assert formatted.endswith("</details>")
  22. def test_remove_triple_backticks():
  23. assert remove_triple_backticks("```") == ""
  24. def test_is_github_repo_url():
  25. assert is_github_repo_url("https://github.com/princeton-nlp/SWE-agent")
  26. assert is_github_repo_url("https://github.com/princeton-nlp/SWE-agent/anything")
  27. assert is_github_repo_url("github.com/princeton-nlp/SWE-agent/anything")
  28. assert not is_github_repo_url("")
  29. assert not is_github_repo_url("/path/to/file")
  30. def test_parse_gh_repo_url():
  31. assert parse_gh_repo_url("https://github.com/princeton-nlp/SWE-agent") == ("princeton-nlp", "SWE-agent")
  32. assert parse_gh_repo_url("github.com/princeton-nlp/SWE-agent") == ("princeton-nlp", "SWE-agent")
  33. assert parse_gh_repo_url("github.com/princeton-nlp/SWE-agent/asdfjsdfg") == ("princeton-nlp", "SWE-agent")
  34. assert parse_gh_repo_url("git@github.com/princeton-nlp/SWE-agent/asdfjsdfg") == ("princeton-nlp", "SWE-agent")
  35. def test_parse_gh_repo_url_fails():
  36. with pytest.raises(InvalidGithubURL):
  37. parse_gh_repo_url("adfkj;lasdfl;kj")
  38. with pytest.raises(InvalidGithubURL):
  39. parse_gh_repo_url("github.com/")
  40. with pytest.raises(InvalidGithubURL):
  41. parse_gh_repo_url("github.com//a/")
  42. def test_parse_gh_issue_url():
  43. url = "https://github.com/princeton-nlp/SWE-agent/issues/43"
  44. owner, repo, no = parse_gh_issue_url(url)
  45. assert owner == "princeton-nlp"
  46. assert repo == "SWE-agent"
  47. assert no == "43"
  48. def test_parse_gh_issue_url_fails():
  49. with pytest.raises(InvalidGithubURL):
  50. parse_gh_issue_url("https://github.com/a/b")
  51. with pytest.raises(InvalidGithubURL):
  52. parse_gh_issue_url("https://github.com/a/b////")
  53. def test_is_from_github_url():
  54. assert not is_github_issue_url("")
  55. assert is_github_issue_url("https://github.com/princeton-nlp/SWE-agent/issues/43")
  56. def test_get_associated_commit_urls():
  57. assoc = get_associated_commit_urls(
  58. org="princeton-nlp",
  59. repo="SWE-agent",
  60. issue_number="41",
  61. token=os.environ.get("GITHUB_TOKEN", ""),
  62. )
  63. assert len(assoc) > 0
  64. def test_get_instance_gh_issue():
  65. instance = get_instances("https://github.com/klieret/swe-agent-test-repo/issues/1", **_TOKEN)[0]
  66. compare_with = {
  67. "repo": "klieret/swe-agent-test-repo",
  68. "instance_id": "klieret__swe-agent-test-repo-i1",
  69. "repo_type": "github",
  70. }
  71. for key in compare_with:
  72. assert instance[key] == compare_with[key]
  73. assert "SyntaxError" in instance["problem_statement"]
  74. assert len(instance["base_commit"]) > 10
  75. assert instance["version"]
  76. def clone_repo(tmp_path, repo_url):
  77. cmd = [
  78. "git",
  79. "clone",
  80. repo_url,
  81. ]
  82. subprocess.run(cmd, check=True, cwd=tmp_path)
  83. def test_get_instance_gh_issue_local_repo(tmp_path):
  84. clone_repo(tmp_path, "https://github.com/klieret/swe-agent-test-repo/")
  85. instance = get_instances(
  86. file_path="https://github.com/klieret/swe-agent-test-repo/issues/1",
  87. repo_path=str(tmp_path / "swe-agent-test-repo"),
  88. **_TOKEN,
  89. )[0]
  90. compare_with = {
  91. "repo": str(tmp_path.resolve() / "swe-agent-test-repo"),
  92. "repo_type": "local",
  93. "instance_id": "klieret__swe-agent-test-repo-i1",
  94. }
  95. for key in compare_with:
  96. assert instance[key] == compare_with[key]
  97. assert "SyntaxError" in instance["problem_statement"]
  98. assert len(instance["base_commit"]) > 10
  99. assert instance["version"]
  100. def test_get_instance_local_issue_local_repo(tmp_path):
  101. clone_repo(tmp_path, "https://github.com/klieret/swe-agent-test-repo/")
  102. issue_path = tmp_path / "issue.txt"
  103. issue_path.write_text("asdf")
  104. instance = get_instances(
  105. file_path=str(issue_path),
  106. repo_path=str(tmp_path / "swe-agent-test-repo"),
  107. )[0]
  108. compare_with = {
  109. "repo": str(tmp_path.resolve() / "swe-agent-test-repo"),
  110. "repo_type": "local",
  111. "instance_id": hashlib.sha256(b"asdf").hexdigest()[:6],
  112. "problem_statement": "asdf",
  113. }
  114. for key in compare_with:
  115. assert instance[key] == compare_with[key]
  116. assert len(instance["base_commit"]) > 10
  117. assert instance["version"]
  118. def test_get_instance_gh_issue_gh_repo(tmp_path):
  119. instance = get_instances(
  120. file_path="https://github.com/klieret/swe-agent-test-repo/issues/1",
  121. repo_path="https://github.com/princeton-nlp/SWE-agent",
  122. **_TOKEN,
  123. )[0]
  124. compare_with = {
  125. "repo": "princeton-nlp/SWE-agent",
  126. "repo_type": "github",
  127. "instance_id": "klieret__swe-agent-test-repo-i1",
  128. }
  129. for key in compare_with:
  130. assert instance[key] == compare_with[key]
  131. assert "SyntaxError" in instance["problem_statement"]
  132. assert len(instance["base_commit"]) > 10
  133. assert instance["version"]
  134. def test_get_instance_text_issue_gh_repo(tmp_path):
  135. instance = get_instances(
  136. file_path="text://this is a test",
  137. repo_path="https://github.com/princeton-nlp/SWE-agent",
  138. **_TOKEN,
  139. )[0]
  140. compare_with = {
  141. "repo": "princeton-nlp/SWE-agent",
  142. "repo_type": "github",
  143. "problem_statement": "this is a test",
  144. }
  145. for key in compare_with:
  146. assert instance[key] == compare_with[key]
  147. assert len(instance["base_commit"]) > 10
  148. assert instance["version"]
  149. def test_load_instances(test_data_path, caplog):
  150. test_data_sources = test_data_path / "data_sources"
  151. examples = list(test_data_sources.iterdir())
  152. for example in examples:
  153. get_instances(file_path=str(example), **_TOKEN)