test_env.py 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. from __future__ import annotations
  2. import dataclasses
  3. import subprocess
  4. import time
  5. from contextlib import contextmanager
  6. from pathlib import Path
  7. from unittest import mock
  8. import pytest
  9. import yaml
  10. import docker
  11. import docker.errors
  12. from sweagent import CONFIG_DIR
  13. from sweagent.environment.swe_env import EnvHook, EnvironmentArguments, SWEEnv
  14. @pytest.fixture(scope="module")
  15. def test_env_args(
  16. tmpdir_factory,
  17. ):
  18. """This will use a persistent container"""
  19. local_repo_path = tmpdir_factory.getbasetemp() / "swe-agent-test-repo"
  20. clone_cmd = ["git", "clone", "https://github.com/klieret/swe-agent-test-repo", local_repo_path]
  21. subprocess.run(clone_cmd, check=True)
  22. data_path = local_repo_path / "problem_statements" / "1.md"
  23. test_env_args = EnvironmentArguments(
  24. data_path=str(data_path),
  25. repo_path=str(local_repo_path),
  26. image_name="sweagent/swe-agent:latest",
  27. container_name="test-container-this-is-a-random-string",
  28. verbose=True,
  29. )
  30. yield test_env_args
  31. # Cleanup (after session ends)
  32. client = docker.from_env()
  33. # fixme (?): What happens if user changed container_name?
  34. try:
  35. container = client.containers.get(test_env_args.container_name)
  36. container.remove(force=True)
  37. except docker.errors.NotFound:
  38. # Can happen if this fixture never runs because we only do a partial
  39. # test run
  40. pass
  41. @contextmanager
  42. def swe_env_context(env_args):
  43. """Context manager to make sure we close the shell on the container
  44. so that we can reuse it.
  45. """
  46. env = SWEEnv(env_args)
  47. try:
  48. yield env
  49. finally:
  50. env.close()
  51. @pytest.mark.slow()
  52. def test_init_swe_env(test_env_args):
  53. with swe_env_context(test_env_args) as env:
  54. env.reset()
  55. @pytest.mark.slow()
  56. def test_init_swe_env_conservative_clone(test_env_args):
  57. with mock.patch.dict("os.environ", {"SWE_AGENT_CLONE_METHOD": "full"}):
  58. with swe_env_context(test_env_args) as env:
  59. env.reset()
  60. @pytest.mark.slow()
  61. def test_init_swe_env_non_persistent(test_env_args):
  62. test_env_args = dataclasses.replace(test_env_args, container_name=None)
  63. with swe_env_context(test_env_args) as env:
  64. env.reset()
  65. @pytest.mark.slow()
  66. def test_init_swe_env_cached_task_image(test_env_args):
  67. test_env_args = dataclasses.replace(test_env_args, cache_task_images=True, container_name=None)
  68. start = time.perf_counter()
  69. with swe_env_context(test_env_args) as env:
  70. env.reset()
  71. duration_no_cache = time.perf_counter() - start
  72. start = time.perf_counter()
  73. # now it should be cached, so let's run again
  74. image_prefix = None
  75. with swe_env_context(test_env_args) as env:
  76. env.reset()
  77. image_prefix = env.cached_image_prefix
  78. assert image_prefix
  79. duration_cache = time.perf_counter() - start
  80. assert duration_cache < duration_no_cache
  81. # Retrieve all images with a prefix "prefix"
  82. client = docker.from_env()
  83. # Remove the images
  84. for image in client.images.list():
  85. if not image.tags:
  86. continue
  87. if not image.tags[0].startswith(image_prefix):
  88. continue
  89. client.images.remove(image.id)
  90. @pytest.mark.slow()
  91. def test_execute_setup_script(tmp_path, test_env_args):
  92. test_script = "echo 'hello world'"
  93. script_path = Path(tmp_path / "test_script.sh")
  94. script_path.write_text(test_script)
  95. test_env_args = dataclasses.replace(test_env_args, environment_setup=script_path)
  96. with swe_env_context(test_env_args) as env:
  97. env.reset()
  98. @pytest.mark.slow()
  99. def test_execute_environment(tmp_path, test_env_args, capsys):
  100. test_env = {
  101. "python": "3.6",
  102. "packages": "pytest",
  103. "pip_packages": ["tox"],
  104. "install": "python -m pip install --upgrade pip && python -m pip install -e .",
  105. }
  106. env_config_path = Path(tmp_path / "env_config.yml")
  107. env_config_path.write_text(yaml.dump(test_env))
  108. # Make sure we don't use persistent container, else we might have already installed the conda environment
  109. test_env_args = dataclasses.replace(test_env_args, environment_setup=env_config_path, container_name=None)
  110. with swe_env_context(test_env_args) as env:
  111. env.reset()
  112. out = capsys.readouterr().out
  113. print(out)
  114. assert "Cloned python conda environment" not in out
  115. @pytest.mark.slow()
  116. def test_execute_environment_default(test_env_args):
  117. env_config_paths = (CONFIG_DIR / "environment_setup").iterdir()
  118. assert env_config_paths
  119. # Make sure we don't use persistent container, else we might have already installed the conda environment
  120. test_env_args = dataclasses.replace(test_env_args, container_name=None)
  121. for env_config_path in env_config_paths:
  122. if env_config_path.name == "django.yaml":
  123. continue
  124. if env_config_path.suffix not in [".yaml", ".yml", ".sh"]:
  125. continue
  126. print(env_config_path)
  127. test_env_args = dataclasses.replace(test_env_args, environment_setup=env_config_path)
  128. with swe_env_context(test_env_args) as env:
  129. env.reset()
  130. @pytest.mark.slow()
  131. def test_execute_environment_clone_python(tmp_path, test_env_args, capsys):
  132. """This should clone the existing python 3.10 conda environment for speedup"""
  133. test_env = {
  134. "python": "3.10",
  135. "packages": "pytest",
  136. "pip_packages": ["tox"],
  137. "install": "python -m pip install --upgrade pip && python -m pip install -e .",
  138. }
  139. env_config_path = Path(tmp_path / "env_config.yml")
  140. env_config_path.write_text(yaml.dump(test_env))
  141. # Make sure we don't use persistent container, else we might have already installed the conda environment
  142. test_env_args = dataclasses.replace(test_env_args, environment_setup=env_config_path, container_name=None)
  143. with swe_env_context(test_env_args) as env:
  144. env.reset()
  145. out = capsys.readouterr().out
  146. print(out)
  147. assert "Cloned python conda environment" in out
  148. @pytest.mark.slow()
  149. def test_open_pr(test_env_args):
  150. test_env_args = dataclasses.replace(
  151. test_env_args,
  152. data_path="https://github.com/klieret/swe-agent-test-repo/issues/1",
  153. repo_path="",
  154. )
  155. with swe_env_context(test_env_args) as env:
  156. env.reset()
  157. env.open_pr(_dry_run=True, trajectory=[])
  158. @pytest.mark.slow()
  159. def test_interrupt_close(test_env_args):
  160. with swe_env_context(test_env_args) as env:
  161. env.reset()
  162. env.interrupt()
  163. @pytest.mark.slow()
  164. def test_communicate_old(test_env_args):
  165. with mock.patch.dict("os.environ", {"SWE_AGENT_COMMUNICATE_METHOD": "processes"}):
  166. with swe_env_context(test_env_args) as env:
  167. env.reset()
  168. @pytest.mark.slow()
  169. def test_env_with_hook(test_env_args):
  170. with swe_env_context(test_env_args) as env:
  171. env.add_hook(EnvHook())
  172. env.reset()
  173. def test_invalid_config():
  174. with pytest.raises(ValueError, match=".*Not allowed.*"):
  175. EnvironmentArguments(
  176. data_path=".",
  177. container_name="test",
  178. cache_task_images=True,
  179. )