interact.py 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. # executable that you can use to upload to the agent hub and download from it
  2. # we also use this to check dependencies to run agents
  3. import argparse
  4. import json
  5. import subprocess
  6. import requests
  7. import gzip
  8. import base64
  9. import sys
  10. import os
  11. class Interactor:
  12. def __init__(self):
  13. script_path = os.path.abspath(__file__)
  14. script_dir = os.path.dirname(script_path)
  15. self.base_folder = script_dir
  16. def list_available_agents(self) -> list[dict]:
  17. """List available agents in the database"""
  18. url = "https://openagi-beta.vercel.app/api/get_all_agents"
  19. response = requests.get(url)
  20. response: dict = response.json()
  21. agent_list = []
  22. for v in list(response.values())[:-1]:
  23. agent_list.append({
  24. "agent": "/".join([v["author"], v["name"]])
  25. })
  26. return agent_list
  27. def download_agent(self, agent: str) -> None:
  28. """Download an agent from the database
  29. Args:
  30. agent (str): in the format of "author/agent_name"
  31. """
  32. assert "/" in agent, 'agent_name should in the format of "author/agent_name"'
  33. author, name = agent.split("/")
  34. # print(author, name)
  35. query = f'https://openagi-beta.vercel.app/api/download?author={author}&name={name}'
  36. response = requests.get(query)
  37. response: dict = response.json()
  38. agent_folder = os.path.join(self.base_folder, agent)
  39. if not os.path.exists(agent_folder):
  40. os.makedirs(agent_folder)
  41. encoded_config = response.get('config')
  42. encoded_code = response.get("code")
  43. encoded_reqs = response.get('dependencies')
  44. if (encoded_config is None) or (encoded_code is None) or (encoded_reqs is None):
  45. print("Agent not found. Try uploading it first?")
  46. return
  47. self.download_config(
  48. self.decompress(encoded_config),
  49. agent
  50. )
  51. self.download_code(
  52. self.decompress(encoded_code),
  53. agent
  54. )
  55. self.download_reqs(
  56. self.decompress(encoded_reqs),
  57. agent
  58. )
  59. def upload_agent(self, agent) -> None:
  60. """Upload an agent to the database
  61. Args:
  62. agent (str): in the format of "author/agent_name"
  63. """
  64. agent_dir = os.path.join(self.base_folder, agent)
  65. author, name = agent.split("/")
  66. config_file = os.path.join(agent_dir, "config.json")
  67. with open(config_file, 'r') as f:
  68. config_data: dict[str, dict] = json.load(f)
  69. meta_data = config_data.get('meta')
  70. headers = { "Content-Type": "application/json" }
  71. # compress python code
  72. encoded_code = self.compress(
  73. self.minify_python_code(agent_dir)
  74. )
  75. # compress meta_requirements.txt
  76. encoded_reqs = self.compress(
  77. self.minify_reqs(agent_dir)
  78. )
  79. # compress config.json
  80. config_str = json.dumps(config_data)
  81. encoded_config = self.compress(config_str)
  82. upload_content = {
  83. 'author': author,
  84. 'name': name,
  85. 'version': meta_data.get('version'),
  86. 'license': meta_data.get('license'),
  87. 'config': encoded_config,
  88. 'code': encoded_code,
  89. 'dependencies': encoded_reqs
  90. }
  91. url = 'https://openagi-beta.vercel.app/api/upload'
  92. response = requests.post(
  93. url,
  94. data=json.dumps(upload_content),
  95. headers=headers
  96. )
  97. if response.content:
  98. print("Uploaded successfully.")
  99. def minify_python_code(self, agent_dir):
  100. code_path = os.path.join(agent_dir, "agent.py")
  101. with open(code_path, 'r') as file:
  102. lines: list[str] = file.readlines()
  103. minified_lines = []
  104. for line in lines:
  105. stripped_line = line.rstrip()
  106. if stripped_line and not stripped_line.lstrip().startswith("#"):
  107. minified_lines.append(stripped_line)
  108. minified_code = "\n".join(minified_lines)
  109. return minified_code
  110. def minify_reqs(self, agent_dir):
  111. req_path = os.path.join(agent_dir, "meta_requirements.txt")
  112. with open(req_path, 'r') as file:
  113. self.reqs: str = file.read()
  114. cleaned = [line.strip() for line in self.reqs.split(
  115. '\n') if line.strip() and not line.startswith('#')]
  116. minified_reqs = ';'.join(cleaned)
  117. return minified_reqs
  118. def minify_config(self, config_data):
  119. minified_config = self.compress(config_data)
  120. return minified_config
  121. def compress(self, minified_data):
  122. compressed_data = gzip.compress(minified_data.encode('utf-8'))
  123. encoded_data = base64.b64encode(compressed_data)
  124. encoded_data = encoded_data.decode('utf-8')
  125. return encoded_data
  126. def decompress(self, encoded_data):
  127. compressed_data = base64.b64decode(encoded_data)
  128. decompressed_data = gzip.decompress(compressed_data)
  129. decompressed_data = decompressed_data.decode("utf-8")
  130. decompressed_data.replace(";", "\n")
  131. return decompressed_data
  132. def download_config(self, config_data, agent) :
  133. config_path = os.path.join(self.base_folder, agent, "config.json")
  134. config_data = json.loads(config_data)
  135. with open(config_path, "w") as w:
  136. json.dump(config_data, w, indent=4)
  137. def download_reqs(self, reqs_data, agent):
  138. reqs_path = os.path.join(self.base_folder, agent, "meta_requirements.txt")
  139. reqs_data = reqs_data.replace(";", "\n")
  140. with open(reqs_path, 'w') as file:
  141. file.write(reqs_data)
  142. def download_code(self, code_data, agent):
  143. code_path = os.path.join(self.base_folder, agent, "agent.py")
  144. with open(code_path, 'w', newline='') as file:
  145. file.write(code_data)
  146. def check_reqs_installed(self, agent):
  147. # Run the `conda list` command and capture the output
  148. reqs_path = os.path.join(self.base_folder, agent, "meta_requirements.txt")
  149. try:
  150. result = subprocess.run(['conda', 'list'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  151. except Exception:
  152. result = subprocess.run(['pip', 'list', '--format=freeze'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  153. # Decode the output from bytes to string
  154. with open(reqs_path, "r") as f:
  155. reqs = []
  156. lines = f.readlines()
  157. for line in lines:
  158. line = line.replace("\n", "")
  159. if "==" in line:
  160. reqs.append(line.split("==")[0])
  161. else:
  162. reqs.append(line)
  163. output = result.stdout.decode('utf-8')
  164. # Extract the list of installed packages
  165. installed_packages = [line.split()[0] for line in output.splitlines() if line]
  166. # Check for each package if it is installed
  167. for req in reqs:
  168. if req not in installed_packages:
  169. return False
  170. return True
  171. def install_agent_reqs(self, agent):
  172. reqs_path = os.path.join(self.base_folder, agent, "meta_requirements.txt")
  173. with open ("deplogs.txt", "a") as f:
  174. subprocess.check_call([
  175. sys.executable,
  176. "-m",
  177. "pip",
  178. "install",
  179. "-r",
  180. reqs_path],
  181. stdout=f,
  182. stderr=f
  183. )
  184. print("Installing dependencies for agent: " + agent + ". Writing to deplogs.txt")
  185. def parse_args():
  186. parser = argparse.ArgumentParser()
  187. parser.add_argument("--mode", choices=["download", "upload"])
  188. parser.add_argument("--agent", required=True)
  189. args = parser.parse_args()
  190. return args
  191. if __name__ == '__main__':
  192. pass
  193. # list_available_agents() # list agents that can be used from db
  194. args = parse_args()
  195. mode = args.mode
  196. agent = args.agent
  197. client = Interactor()
  198. # client.check_reqs_installed(agent)
  199. # client = Interactor()
  200. if mode == "download":
  201. client.download_agent(agent) # download agents
  202. else:
  203. assert mode == "upload"
  204. client.upload_agent(agent) # upload agents