ray_champagne.py 3.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889
  1. import click
  2. import os
  3. from pathlib import Path
  4. import boto3
  5. from anyscale.sdk.anyscale_client.models.create_cluster_environment import (
  6. CreateClusterEnvironment,
  7. )
  8. from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
  9. from ray_release.bazel import bazel_runfile
  10. from ray_release.byod.build import build_champagne_image
  11. from ray_release.logger import logger
  12. from ray_release.configs.global_config import init_global_config
  13. CHAMPAGNE_IMAGE_TYPES = [
  14. # python_version, image_type, cluster_env_name
  15. ("py38", "cpu", "ray-champagne-cpu"),
  16. ("py38", "gpu", "ray-champagne-gpu"),
  17. ]
  18. ANYSCALE_SECRET_ARM = (
  19. "arn:aws:secretsmanager:us-west-2:029272617770:secret:release-automation"
  20. )
  21. ANYSCALE_SECRETS = {
  22. "anyscale-staging-token20231008005227440600000001-JTgxb0": (
  23. "https://console.anyscale-staging.com"
  24. ),
  25. "anyscale-demos-FaVACh": "https://console.anyscale.com",
  26. }
  27. CONFIG_CHOICES = click.Choice(
  28. [x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
  29. )
  30. @click.option(
  31. "--global-config",
  32. default="oss_config.yaml",
  33. type=CONFIG_CHOICES,
  34. help="Global config to use for test execution.",
  35. )
  36. def main(global_config: str = "oss_config.yaml") -> None:
  37. """
  38. Builds the Ray champagne image.
  39. """
  40. init_global_config(
  41. bazel_runfile("release/ray_release/configs", global_config),
  42. )
  43. branch = os.environ.get("BRANCH_TO_TEST", os.environ["BUILDKITE_BRANCH"])
  44. commit = os.environ.get("COMMIT_TO_TEST", os.environ["BUILDKITE_COMMIT"])
  45. assert branch.startswith(
  46. "releases/"
  47. ), f"Champagne building only supported on release branch, found {branch}"
  48. ray_version = f"{branch[len('releases/') :]}.{commit[:6]}"
  49. for python_version, image_type, cluster_env_name in CHAMPAGNE_IMAGE_TYPES:
  50. logger.info(f"Building champagne image for {python_version} {image_type}")
  51. anyscale_image = build_champagne_image(
  52. ray_version,
  53. python_version,
  54. image_type,
  55. )
  56. logger.info(f"Updating cluster environment {cluster_env_name}")
  57. _build_champaign_cluster_environment(anyscale_image, cluster_env_name)
  58. def _build_champaign_cluster_environment(
  59. anyscale_image: str,
  60. cluster_env_name: str,
  61. ) -> None:
  62. boto = boto3.client("secretsmanager", region_name="us-west-2")
  63. for secret_name, host in ANYSCALE_SECRETS.items():
  64. logger.info(
  65. f"\tUpdating cluster environment for secret: {secret_name}, host: {host}"
  66. )
  67. os.environ["ANYSCALE_CLI_TOKEN"] = boto.get_secret_value(
  68. SecretId=f"{ANYSCALE_SECRET_ARM}/{secret_name}"
  69. )["SecretString"]
  70. AnyscaleSDK(host=host).build_cluster_environment(
  71. create_cluster_environment=CreateClusterEnvironment(
  72. name=cluster_env_name,
  73. config_json=dict(
  74. docker_image=anyscale_image,
  75. ray_version="nightly",
  76. env_vars={},
  77. ),
  78. ),
  79. )
  80. if __name__ == "__main__":
  81. main()