template.py 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126
  1. import copy
  2. import datetime
  3. import os
  4. from typing import Optional, Dict, TYPE_CHECKING
  5. import jinja2
  6. import yaml
  7. from ray_release.bazel import bazel_runfile
  8. from ray_release.config import get_test_cloud_id
  9. from ray_release.exception import ReleaseTestConfigError
  10. if TYPE_CHECKING:
  11. from ray_release.config import Test
  12. DEFAULT_ENV = {
  13. "DATESTAMP": str(datetime.datetime.now().strftime("%Y%m%d")),
  14. "TIMESTAMP": str(int(datetime.datetime.now().timestamp())),
  15. }
  16. class TestEnvironment(dict):
  17. pass
  18. _test_env = None
  19. def get_test_environment():
  20. global _test_env
  21. if _test_env:
  22. return _test_env
  23. _test_env = TestEnvironment(**DEFAULT_ENV)
  24. return _test_env
  25. def set_test_env_var(key: str, value: str):
  26. test_env = get_test_environment()
  27. test_env[key] = value
  28. def get_test_env_var(key: str, default: Optional[str] = None):
  29. test_env = get_test_environment()
  30. return test_env.get(key, default)
  31. def get_wheels_sanity_check(commit: Optional[str] = None):
  32. if not commit:
  33. cmd = (
  34. "python -c 'import ray; print("
  35. '"No commit sanity check available, but this is the '
  36. "Ray wheel commit:\", ray.__commit__)'"
  37. )
  38. else:
  39. cmd = (
  40. f"python -c 'import ray; "
  41. f'assert ray.__commit__ == "{commit}", ray.__commit__\''
  42. )
  43. return cmd
  44. def load_and_render_yaml_template(
  45. template_path: str, env: Optional[Dict] = None
  46. ) -> Optional[Dict]:
  47. if not template_path:
  48. return None
  49. if not os.path.isfile(template_path):
  50. raise ReleaseTestConfigError(
  51. f"Cannot load yaml template from {template_path}: Path not found."
  52. )
  53. with open(template_path, "rt") as f:
  54. content = f.read()
  55. return render_yaml_template(template=content, env=env)
  56. def render_yaml_template(template: str, env: Optional[Dict] = None):
  57. render_env = copy.deepcopy(os.environ)
  58. if env:
  59. render_env.update(env)
  60. try:
  61. content = jinja2.Template(template).render(env=render_env)
  62. return yaml.safe_load(content)
  63. except Exception as e:
  64. raise ReleaseTestConfigError(
  65. f"Error rendering/loading yaml template: {e}"
  66. ) from e
  67. def get_cluster_env_path(
  68. test: "Test", test_definition_root: Optional[str] = None
  69. ) -> str:
  70. working_dir = test.get("working_dir", "")
  71. cluster_env_file = test["cluster"]["cluster_env"]
  72. return (
  73. os.path.join(test_definition_root, working_dir, cluster_env_file)
  74. if test_definition_root
  75. else bazel_runfile("release", working_dir, cluster_env_file)
  76. )
  77. def load_test_cluster_compute(
  78. test: "Test", test_definition_root: Optional[str] = None
  79. ) -> Optional[Dict]:
  80. cluster_compute_file = test["cluster"]["cluster_compute"]
  81. working_dir = test.get("working_dir", "")
  82. f = (
  83. os.path.join(test_definition_root, working_dir, cluster_compute_file)
  84. if test_definition_root
  85. else bazel_runfile("release", working_dir, cluster_compute_file)
  86. )
  87. env = populate_cluster_compute_variables(test)
  88. return load_and_render_yaml_template(f, env=env)
  89. def populate_cluster_compute_variables(test: "Test") -> Dict:
  90. env = get_test_environment()
  91. cloud_id = get_test_cloud_id(test)
  92. env["ANYSCALE_CLOUD_ID"] = cloud_id
  93. return env