123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137 |
- import collections
- import hashlib
- import json
- import os
- import subprocess
- import time
- from typing import Dict, Any, List, Tuple
- import requests
- from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
- from ray_release.logger import logger
- ANYSCALE_HOST = os.environ.get("ANYSCALE_HOST", "https://console.anyscale.com")
- def deep_update(d, u) -> Dict:
- for k, v in u.items():
- if isinstance(v, collections.abc.Mapping):
- d[k] = deep_update(d.get(k, {}), v)
- else:
- d[k] = v
- return d
- def dict_hash(dt: Dict[Any, Any]) -> str:
- json_str = json.dumps(dt, sort_keys=True, ensure_ascii=True)
- sha = hashlib.sha256()
- sha.update(json_str.encode())
- return sha.hexdigest()
- def url_exists(url: str) -> bool:
- return requests.head(url, allow_redirects=True).status_code == 200
- def resolve_url(url: str) -> str:
- return requests.head(url, allow_redirects=True).url
- def format_link(link: str) -> str:
- # Use ANSI escape code to allow link to be clickable
- # https://buildkite.com/docs/pipelines/links-and-images
- # -in-log-output
- if os.environ.get("BUILDKITE_COMMIT"):
- return "\033]1339;url='" + link + "'\a\n"
- # Else, no buildkite:
- return link
- def anyscale_project_url(project_id: str) -> str:
- return (
- f"{ANYSCALE_HOST}"
- f"/o/anyscale-internal/projects/{project_id}"
- f"/?tab=session-list"
- )
- def anyscale_cluster_url(project_id: str, session_id: str) -> str:
- return (
- f"{ANYSCALE_HOST}"
- f"/o/anyscale-internal/projects/{project_id}"
- f"/clusters/{session_id}"
- )
- def anyscale_cluster_compute_url(compute_tpl_id: str) -> str:
- return (
- f"{ANYSCALE_HOST}"
- f"/o/anyscale-internal/configurations/cluster-computes"
- f"/{compute_tpl_id}"
- )
- def anyscale_cluster_env_build_url(build_id: str) -> str:
- return (
- f"{ANYSCALE_HOST}"
- f"/o/anyscale-internal/configurations/app-config-details"
- f"/{build_id}"
- )
- _anyscale_sdk = None
- def get_anyscale_sdk() -> AnyscaleSDK:
- global _anyscale_sdk
- if _anyscale_sdk:
- return _anyscale_sdk
- _anyscale_sdk = AnyscaleSDK()
- return _anyscale_sdk
- def exponential_backoff_retry(
- f, retry_exceptions, initial_retry_delay_s, max_retries
- ) -> None:
- retry_cnt = 0
- retry_delay_s = initial_retry_delay_s
- while True:
- try:
- return f()
- except retry_exceptions as e:
- retry_cnt += 1
- if retry_cnt > max_retries:
- raise
- logger.info(
- f"Retry function call failed due to {e} "
- f"in {retry_delay_s} seconds..."
- )
- time.sleep(retry_delay_s)
- retry_delay_s *= 2
- def run_bash_script(bash_script: str) -> None:
- subprocess.run(f"bash {bash_script}", shell=True, check=True)
- def reinstall_anyscale_dependencies() -> None:
- logger.info("Re-installing `anyscale` package")
- subprocess.check_output(
- "pip install -U anyscale",
- shell=True,
- text=True,
- )
- def get_pip_packages() -> List[str]:
- from pip._internal.operations import freeze
- return list(freeze.freeze())
- def python_version_str(python_version: Tuple[int, int]) -> str:
- """From (X, Y) to XY"""
- return "".join([str(x) for x in python_version])
|