run_replay.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. """Replay a trajectory"""
  2. import json
  3. import os
  4. import subprocess
  5. import yaml
  6. from argparse import ArgumentParser
  7. from sweagent.environment.utils import is_from_github_url
  8. from typing import Any, Dict, List
  9. import run as runscript
  10. def process_single_traj(traj_path: str, config_file: str, data_path: str, suffix: str, *, forward_args: List[str]):
  11. """
  12. Args:
  13. traj_path (str): _description_
  14. config_file (str): _description_
  15. data_path (str): _description_
  16. suffix (str): _description_
  17. forward_args (List[str]): Passed to run.py
  18. Raises:
  19. ValueError: Incorrect paths or other config issue
  20. Returns:
  21. None
  22. """
  23. replay_action_trajs_path = "temp_replay.jsonl"
  24. # Open trajectory file, extract responses as actions
  25. if traj_path.endswith(".yaml"):
  26. traj_data = dict()
  27. with open(traj_path, "r") as f:
  28. traj_data["history"] = yaml.safe_load(f)
  29. else:
  30. traj_data = json.load(open(traj_path, "r"))
  31. actions = [x["content"] for x in traj_data["history"] if x["role"] == "assistant"]
  32. instance_id = traj_path.split("/")[-1].split(".")[0]
  33. with open(replay_action_trajs_path, "w") as f:
  34. print(
  35. json.dumps({instance_id: actions}),
  36. file=f,
  37. end="\n",
  38. flush=True
  39. )
  40. # Get data_path from args.yaml
  41. if data_path is None:
  42. args_path = os.path.join(
  43. os.path.dirname(traj_path),
  44. "args.yaml"
  45. )
  46. args = yaml.safe_load(open(args_path))
  47. data_path = args['environment']['data_path']
  48. # Identify the relevant task instance and create it
  49. def create_task_instances_tmp_file(data: List[Dict[str, Any]]) -> str:
  50. """Helper function to create a temporary file to write task instances to.
  51. Returns path to the temporary file.
  52. """
  53. data = [d for d in data if d["instance_id"] == instance_id]
  54. tmp_path = instance_id + ".jsonl"
  55. with open(tmp_path, "w") as f:
  56. for d in data:
  57. print(json.dumps(d), file=f, end="\n", flush=True)
  58. return replay_task_instances_path
  59. is_github = False
  60. if data_path.endswith(".jsonl"):
  61. replay_task_instances_path = create_task_instances_tmp_file([json.loads(x) for x in open(data_path, "r").readlines()])
  62. elif data_path.endswith(".json"):
  63. replay_task_instances_path = create_task_instances_tmp_file(json.load(open(data_path)))
  64. elif is_from_github_url(data_path):
  65. is_github = True
  66. replay_task_instances_path = data_path
  67. else:
  68. raise ValueError("--data_path must be a .json or .jsonl")
  69. # Call run.py via subprocess
  70. run_args = [
  71. "--config_file", config_file,
  72. "--data_path", replay_task_instances_path,
  73. "--install_environment", "True",
  74. "--model_name", "replay",
  75. "--replay_path", replay_action_trajs_path,
  76. *forward_args,
  77. ]
  78. if is_github:
  79. # Not sure if this only applies to github urls for data_path
  80. run_args.extend(["--skip_existing", "False"])
  81. if suffix is not None:
  82. run_args.extend(["--suffix", suffix])
  83. script_args = runscript.get_args(run_args)
  84. runscript.main(script_args)
  85. os.remove(replay_action_trajs_path)
  86. try:
  87. os.remove(replay_task_instances_path)
  88. except FileNotFoundError:
  89. pass
  90. def main(
  91. traj_path: str,
  92. config_file: str,
  93. data_path: str,
  94. suffix: str,
  95. *,
  96. forward_args: List[str],
  97. ):
  98. process_single_traj(traj_path, config_file, data_path, suffix, forward_args=forward_args)
  99. def get_args(args=None):
  100. parser = ArgumentParser(description=__doc__)
  101. parser.add_argument("--traj_path", help="Path to trajectory to replay", default=None)
  102. parser.add_argument("--config_file", help="Path to template", required=True)
  103. parser.add_argument("--data_path", help="(Optional) Path to data file containing task instances ref'ed by replay trajectories", default=None)
  104. parser.add_argument("--suffix", help="(Optional) Suffix argument appended to end of traj path", default=None)
  105. args, remaining_args = parser.parse_known_args(args=args)
  106. return args, remaining_args
  107. if __name__ == "__main__":
  108. args, remaining_args = get_args()
  109. main(**vars(args), forward_args=remaining_args)