util.py 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137
  1. import collections
  2. import hashlib
  3. import json
  4. import os
  5. import subprocess
  6. import time
  7. from typing import Dict, Any, List, Tuple
  8. import requests
  9. from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
  10. from ray_release.logger import logger
  11. ANYSCALE_HOST = os.environ.get("ANYSCALE_HOST", "https://console.anyscale.com")
  12. def deep_update(d, u) -> Dict:
  13. for k, v in u.items():
  14. if isinstance(v, collections.abc.Mapping):
  15. d[k] = deep_update(d.get(k, {}), v)
  16. else:
  17. d[k] = v
  18. return d
  19. def dict_hash(dt: Dict[Any, Any]) -> str:
  20. json_str = json.dumps(dt, sort_keys=True, ensure_ascii=True)
  21. sha = hashlib.sha256()
  22. sha.update(json_str.encode())
  23. return sha.hexdigest()
  24. def url_exists(url: str) -> bool:
  25. return requests.head(url, allow_redirects=True).status_code == 200
  26. def resolve_url(url: str) -> str:
  27. return requests.head(url, allow_redirects=True).url
  28. def format_link(link: str) -> str:
  29. # Use ANSI escape code to allow link to be clickable
  30. # https://buildkite.com/docs/pipelines/links-and-images
  31. # -in-log-output
  32. if os.environ.get("BUILDKITE_COMMIT"):
  33. return "\033]1339;url='" + link + "'\a\n"
  34. # Else, no buildkite:
  35. return link
  36. def anyscale_project_url(project_id: str) -> str:
  37. return (
  38. f"{ANYSCALE_HOST}"
  39. f"/o/anyscale-internal/projects/{project_id}"
  40. f"/?tab=session-list"
  41. )
  42. def anyscale_cluster_url(project_id: str, session_id: str) -> str:
  43. return (
  44. f"{ANYSCALE_HOST}"
  45. f"/o/anyscale-internal/projects/{project_id}"
  46. f"/clusters/{session_id}"
  47. )
  48. def anyscale_cluster_compute_url(compute_tpl_id: str) -> str:
  49. return (
  50. f"{ANYSCALE_HOST}"
  51. f"/o/anyscale-internal/configurations/cluster-computes"
  52. f"/{compute_tpl_id}"
  53. )
  54. def anyscale_cluster_env_build_url(build_id: str) -> str:
  55. return (
  56. f"{ANYSCALE_HOST}"
  57. f"/o/anyscale-internal/configurations/app-config-details"
  58. f"/{build_id}"
  59. )
  60. _anyscale_sdk = None
  61. def get_anyscale_sdk() -> AnyscaleSDK:
  62. global _anyscale_sdk
  63. if _anyscale_sdk:
  64. return _anyscale_sdk
  65. _anyscale_sdk = AnyscaleSDK()
  66. return _anyscale_sdk
  67. def exponential_backoff_retry(
  68. f, retry_exceptions, initial_retry_delay_s, max_retries
  69. ) -> None:
  70. retry_cnt = 0
  71. retry_delay_s = initial_retry_delay_s
  72. while True:
  73. try:
  74. return f()
  75. except retry_exceptions as e:
  76. retry_cnt += 1
  77. if retry_cnt > max_retries:
  78. raise
  79. logger.info(
  80. f"Retry function call failed due to {e} "
  81. f"in {retry_delay_s} seconds..."
  82. )
  83. time.sleep(retry_delay_s)
  84. retry_delay_s *= 2
  85. def run_bash_script(bash_script: str) -> None:
  86. subprocess.run(f"bash {bash_script}", shell=True, check=True)
  87. def reinstall_anyscale_dependencies() -> None:
  88. logger.info("Re-installing `anyscale` package")
  89. subprocess.check_output(
  90. "pip install -U anyscale",
  91. shell=True,
  92. text=True,
  93. )
  94. def get_pip_packages() -> List[str]:
  95. from pip._internal.operations import freeze
  96. return list(freeze.freeze())
  97. def python_version_str(python_version: Tuple[int, int]) -> str:
  98. """From (X, Y) to XY"""
  99. return "".join([str(x) for x in python_version])