__init__.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. # isort: skip_file
  2. import logging
  3. import os
  4. import sys
  5. logger = logging.getLogger(__name__)
  6. def _configure_system():
  7. import os
  8. import platform
  9. import sys
  10. """Wraps system configuration to avoid 'leaking' variables into ray."""
  11. # Sanity check pickle5 if it has been installed.
  12. if "pickle5" in sys.modules:
  13. if sys.version_info >= (3, 8):
  14. logger.warning(
  15. "Package pickle5 becomes unnecessary in Python 3.8 and above. "
  16. "Its presence may confuse libraries including Ray. "
  17. "Please uninstall the package."
  18. )
  19. import pkg_resources
  20. try:
  21. version_info = pkg_resources.require("pickle5")
  22. version = tuple(int(n) for n in version_info[0].version.split("."))
  23. if version < (0, 0, 10):
  24. logger.warning(
  25. "Although not used by Ray, a version of pickle5 that leaks memory "
  26. "is found in the environment. Please run 'pip install pickle5 -U' "
  27. "to upgrade."
  28. )
  29. except pkg_resources.DistributionNotFound:
  30. logger.warning(
  31. "You are using the 'pickle5' module, but "
  32. "the exact version is unknown (possibly carried as "
  33. "an internal component by another module). Please "
  34. "make sure you are using pickle5 >= 0.0.10 because "
  35. "previous versions may leak memory."
  36. )
  37. # MUST add pickle5 to the import path because it will be imported by some
  38. # raylet modules.
  39. #
  40. # When running Python version < 3.8, Ray needs to use pickle5 instead of
  41. # Python's built-in pickle. Add the directory containing pickle5 to the
  42. # Python path so that we find the pickle5 version packaged with Ray and
  43. # not a pre-existing pickle5.
  44. if sys.version_info < (3, 8):
  45. pickle5_path = os.path.join(
  46. os.path.abspath(os.path.dirname(__file__)), "pickle5_files"
  47. )
  48. sys.path.insert(0, pickle5_path)
  49. # Check that grpc can actually be imported on Apple Silicon. Some package
  50. # managers (such as `pip`) can't properly install the grpcio library yet,
  51. # so provide a proactive error message if that's the case.
  52. if platform.system() == "Darwin" and platform.machine() == "arm64":
  53. try:
  54. import grpc # noqa: F401
  55. except ImportError:
  56. raise ImportError(
  57. "Failed to import grpc on Apple Silicon. On Apple"
  58. " Silicon machines, try `pip uninstall grpcio; conda "
  59. "install grpcio`. Check out "
  60. "https://docs.ray.io/en/master/ray-overview/installation.html"
  61. "#m1-mac-apple-silicon-support for more details."
  62. )
  63. # Importing psutil & setproctitle. Must be before ray._raylet is
  64. # initialized.
  65. thirdparty_files = os.path.join(
  66. os.path.abspath(os.path.dirname(__file__)), "thirdparty_files"
  67. )
  68. sys.path.insert(0, thirdparty_files)
  69. if (
  70. platform.system() == "Linux"
  71. and "Microsoft".lower() in platform.release().lower()
  72. ):
  73. import ray._private.compat # noqa: E402
  74. ray._private.compat.patch_psutil()
  75. # Expose ray ABI symbols which may be dependent by other shared
  76. # libraries such as _streaming.so. See BUILD.bazel:_raylet
  77. python_shared_lib_suffix = ".so" if sys.platform != "win32" else ".pyd"
  78. so_path = os.path.join(
  79. os.path.dirname(__file__), "_raylet" + python_shared_lib_suffix
  80. )
  81. if os.path.exists(so_path):
  82. import ctypes
  83. from ctypes import CDLL
  84. CDLL(so_path, ctypes.RTLD_GLOBAL)
  85. _configure_system()
  86. # Delete configuration function.
  87. del _configure_system
  88. # Replaced with the current commit when building the wheels.
  89. __commit__ = "{{RAY_COMMIT_SHA}}"
  90. __version__ = "3.0.0.dev0"
  91. import ray._raylet # noqa: E402
  92. from ray._raylet import ( # noqa: E402,F401
  93. ActorClassID,
  94. ActorID,
  95. NodeID,
  96. Config as _Config,
  97. JobID,
  98. WorkerID,
  99. FunctionID,
  100. ObjectID,
  101. ObjectRef,
  102. ObjectRefGenerator,
  103. TaskID,
  104. UniqueID,
  105. Language,
  106. PlacementGroupID,
  107. )
  108. _config = _Config()
  109. from ray._private.state import ( # noqa: E402,F401
  110. nodes,
  111. timeline,
  112. cluster_resources,
  113. available_resources,
  114. )
  115. from ray._private.worker import ( # noqa: E402,F401
  116. LOCAL_MODE,
  117. SCRIPT_MODE,
  118. WORKER_MODE,
  119. RESTORE_WORKER_MODE,
  120. SPILL_WORKER_MODE,
  121. cancel,
  122. get,
  123. get_actor,
  124. get_gpu_ids,
  125. init,
  126. is_initialized,
  127. put,
  128. kill,
  129. remote,
  130. shutdown,
  131. wait,
  132. )
  133. # We import ray.actor because some code is run in actor.py which initializes
  134. # some functions in the worker.
  135. import ray.actor # noqa: E402,F401
  136. from ray.actor import method # noqa: E402,F401
  137. # TODO(qwang): We should remove this exporting in Ray2.0.
  138. from ray.cross_language import java_function, java_actor_class # noqa: E402,F401
  139. from ray.runtime_context import get_runtime_context # noqa: E402,F401
  140. from ray import autoscaler # noqa: E402,F401
  141. from ray import internal # noqa: E402,F401
  142. from ray import util # noqa: E402,F401
  143. from ray import _private # noqa: E402,F401
  144. # We import ClientBuilder so that modules can inherit from `ray.ClientBuilder`.
  145. from ray.client_builder import client, ClientBuilder # noqa: E402,F401
  146. class _DeprecationWrapper:
  147. def __init__(self, name, real_worker):
  148. self._name = name
  149. self._real_worker = real_worker
  150. self._warned = set()
  151. def __getattr__(self, attr):
  152. value = getattr(self._real_worker, attr)
  153. if attr not in self._warned:
  154. self._warned.add(attr)
  155. logger.warning(
  156. f"DeprecationWarning: `ray.{self._name}.{attr}` is a private "
  157. "attribute and access will be removed in a future Ray version."
  158. )
  159. return value
  160. # TODO(ekl) remove this entirely after 3rd party libraries are all migrated.
  161. worker = _DeprecationWrapper("worker", ray._private.worker)
  162. ray_constants = _DeprecationWrapper("ray_constants", ray._private.ray_constants)
  163. serialization = _DeprecationWrapper("serialization", ray._private.serialization)
  164. state = _DeprecationWrapper("state", ray._private.state)
  165. __all__ = [
  166. "__version__",
  167. "_config",
  168. "get_runtime_context",
  169. "actor",
  170. "autoscaler",
  171. "available_resources",
  172. "cancel",
  173. "client",
  174. "ClientBuilder",
  175. "cluster_resources",
  176. "get",
  177. "get_actor",
  178. "get_gpu_ids",
  179. "init",
  180. "internal",
  181. "is_initialized",
  182. "java_actor_class",
  183. "java_function",
  184. "cpp_function",
  185. "kill",
  186. "Language",
  187. "method",
  188. "nodes",
  189. "put",
  190. "remote",
  191. "shutdown",
  192. "show_in_dashboard",
  193. "timeline",
  194. "util",
  195. "wait",
  196. "widgets",
  197. "LOCAL_MODE",
  198. "SCRIPT_MODE",
  199. "WORKER_MODE",
  200. ]
  201. # Subpackages
  202. __all__ += [
  203. "data",
  204. "workflow",
  205. ]
  206. # ID types
  207. __all__ += [
  208. "ActorClassID",
  209. "ActorID",
  210. "NodeID",
  211. "JobID",
  212. "WorkerID",
  213. "FunctionID",
  214. "ObjectID",
  215. "ObjectRef",
  216. "ObjectRefGenerator",
  217. "TaskID",
  218. "UniqueID",
  219. "PlacementGroupID",
  220. ]
  221. if sys.version_info < (3, 7):
  222. # TODO(Clark): Remove this one we drop Python 3.6 support.
  223. from ray import data # noqa: F401
  224. from ray import workflow # noqa: F401
  225. else:
  226. # Delay importing of expensive, isolated subpackages.
  227. def __getattr__(name: str):
  228. import importlib
  229. if name in ["data", "workflow"]:
  230. return importlib.import_module("." + name, __name__)
  231. raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
  232. del os
  233. del logging
  234. del sys