run.py 1.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. import subprocess
  2. import click
  3. import json
  4. import os
  5. import time
  6. from logger import logger
  7. WORKLOAD_SCRIPTS = [
  8. "test_core.py",
  9. ]
  10. def setup_cluster():
  11. from ray.cluster_utils import AutoscalingCluster
  12. cluster = AutoscalingCluster(
  13. head_resources={"CPU": 0},
  14. worker_node_types={
  15. "type-1": {
  16. "resources": {"CPU": 4},
  17. "node_config": {},
  18. "min_workers": 0,
  19. "max_workers": 10,
  20. },
  21. },
  22. idle_timeout_minutes=1 * 0.1,
  23. )
  24. cluster.start(_system_config={"enable_autoscaler_v2": True})
  25. return cluster
  26. def run_test():
  27. failed_workloads = []
  28. for workload in WORKLOAD_SCRIPTS:
  29. # Run the python script.
  30. logger.info(f"Running workload {workload}:")
  31. try:
  32. subprocess.check_call(["python", workload])
  33. except subprocess.CalledProcessError as e:
  34. failed_workloads.append((workload, e))
  35. if failed_workloads:
  36. for workload, e in failed_workloads:
  37. logger.error(f"Workload {workload} failed with {e}")
  38. else:
  39. logger.info("All workloads passed!")
  40. @click.command()
  41. @click.option("--local", is_flag=True, help="Run locally.", default=False)
  42. def run(local):
  43. start_time = time.time()
  44. cluster = None
  45. try:
  46. if local:
  47. cluster = setup_cluster()
  48. run_test()
  49. cluster.shutdown()
  50. else:
  51. run_test()
  52. success = "1"
  53. except Exception as e:
  54. logger.error(f"Test failed with {e}")
  55. success = "0"
  56. finally:
  57. if cluster:
  58. cluster.shutdown()
  59. results = {
  60. "time": time.time() - start_time,
  61. "success": success,
  62. }
  63. if "TEST_OUTPUT_JSON" in os.environ:
  64. with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
  65. json.dump(results, out_file)
  66. print(json.dumps(results, indent=2))
  67. if __name__ == "__main__":
  68. run()