ray_deps_setup.bzl 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
  2. load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
  3. def urlsplit(url):
  4. """ Splits a URL like "https://example.com/a/b?c=d&e#f" into a tuple:
  5. ("https", ["example", "com"], ["a", "b"], ["c=d", "e"], "f")
  6. A trailing slash will result in a correspondingly empty final path component.
  7. """
  8. split_on_anchor = url.split("#", 1)
  9. split_on_query = split_on_anchor[0].split("?", 1)
  10. split_on_scheme = split_on_query[0].split("://", 1)
  11. if len(split_on_scheme) <= 1: # Scheme is optional
  12. split_on_scheme = [None] + split_on_scheme[:1]
  13. split_on_path = split_on_scheme[1].split("/")
  14. return {
  15. "scheme": split_on_scheme[0],
  16. "netloc": split_on_path[0].split("."),
  17. "path": split_on_path[1:],
  18. "query": split_on_query[1].split("&") if len(split_on_query) > 1 else None,
  19. "fragment": split_on_anchor[1] if len(split_on_anchor) > 1 else None,
  20. }
  21. def auto_http_archive(
  22. *,
  23. name = None,
  24. url = None,
  25. urls = True,
  26. build_file = None,
  27. build_file_content = None,
  28. strip_prefix = True,
  29. **kwargs):
  30. """ Intelligently choose mirrors based on the given URL for the download.
  31. Either url or urls is required.
  32. If name == None , it is auto-deduced, but this is NOT recommended.
  33. If urls == True , mirrors are automatically chosen.
  34. If build_file == True , it is auto-deduced.
  35. If strip_prefix == True , it is auto-deduced.
  36. """
  37. DOUBLE_SUFFIXES_LOWERCASE = [("tar", "bz2"), ("tar", "gz"), ("tar", "xz")]
  38. mirror_prefixes = ["https://mirror.bazel.build/"]
  39. canonical_url = url if url != None else urls[0]
  40. url_parts = urlsplit(canonical_url)
  41. url_except_scheme = (canonical_url.replace(url_parts["scheme"] + "://", "") if url_parts["scheme"] != None else canonical_url)
  42. url_path_parts = url_parts["path"]
  43. url_filename = url_path_parts[-1]
  44. url_filename_parts = (url_filename.rsplit(".", 2) if (tuple(url_filename.lower().rsplit(".", 2)[-2:]) in
  45. DOUBLE_SUFFIXES_LOWERCASE) else url_filename.rsplit(".", 1))
  46. is_github = url_parts["netloc"] == ["github", "com"]
  47. if name == None: # Deduce "com_github_user_project_name" from "https://github.com/user/project-name/..."
  48. name = "_".join(url_parts["netloc"][::-1] + url_path_parts[:2]).replace("-", "_")
  49. # auto appending ray project namespace prefix for 3rd party library reusing.
  50. if build_file == True:
  51. build_file = "@com_github_ray_project_ray//%s:%s" % ("bazel", "BUILD." + name)
  52. if urls == True:
  53. prefer_url_over_mirrors = is_github
  54. urls = [
  55. mirror_prefix + url_except_scheme
  56. for mirror_prefix in mirror_prefixes
  57. if not canonical_url.startswith(mirror_prefix)
  58. ]
  59. urls.insert(0 if prefer_url_over_mirrors else len(urls), canonical_url)
  60. else:
  61. print("No implicit mirrors used because urls were explicitly provided")
  62. if strip_prefix == True:
  63. prefix_without_v = url_filename_parts[0]
  64. if prefix_without_v.startswith("v") and prefix_without_v[1:2].isdigit():
  65. # GitHub automatically strips a leading 'v' in version numbers
  66. prefix_without_v = prefix_without_v[1:]
  67. strip_prefix = (url_path_parts[1] + "-" + prefix_without_v if is_github and url_path_parts[2:3] == ["archive"] else url_filename_parts[0])
  68. return http_archive(
  69. name = name,
  70. url = url,
  71. urls = urls,
  72. build_file = build_file,
  73. build_file_content = build_file_content,
  74. strip_prefix = strip_prefix,
  75. **kwargs
  76. )
  77. def ray_deps_setup():
  78. # Explicitly bring in protobuf dependency to work around
  79. # https://github.com/ray-project/ray/issues/14117
  80. http_archive(
  81. name = "com_google_protobuf",
  82. strip_prefix = "protobuf-3.16.0",
  83. urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.16.0.tar.gz"],
  84. sha256 = "7892a35d979304a404400a101c46ce90e85ec9e2a766a86041bb361f626247f5",
  85. )
  86. # NOTE(lingxuan.zlx): 3rd party dependencies could be accessed, so it suggests
  87. # all of http/git_repository should add prefix for patches defined in ray directory.
  88. auto_http_archive(
  89. name = "com_github_antirez_redis",
  90. build_file = "@com_github_ray_project_ray//bazel:BUILD.redis",
  91. url = "https://github.com/redis/redis/archive/6.0.10.tar.gz",
  92. sha256 = "900cb82227bac58242c9b7668e7113cd952253b256fe04bbdab1b78979cf255a",
  93. patches = [
  94. "@com_github_ray_project_ray//thirdparty/patches:redis-quiet.patch",
  95. ],
  96. )
  97. auto_http_archive(
  98. name = "com_github_redis_hiredis",
  99. build_file = "@com_github_ray_project_ray//bazel:BUILD.hiredis",
  100. url = "https://github.com/redis/hiredis/archive/392de5d7f97353485df1237872cb682842e8d83f.tar.gz",
  101. sha256 = "2101650d39a8f13293f263e9da242d2c6dee0cda08d343b2939ffe3d95cf3b8b",
  102. patches = [
  103. "@com_github_ray_project_ray//thirdparty/patches:hiredis-windows-msvc.patch",
  104. ],
  105. )
  106. auto_http_archive(
  107. name = "com_github_spdlog",
  108. build_file = "@com_github_ray_project_ray//bazel:BUILD.spdlog",
  109. urls = ["https://github.com/gabime/spdlog/archive/v1.7.0.zip"],
  110. sha256 = "c8f1e1103e0b148eb8832275d8e68036f2fdd3975a1199af0e844908c56f6ea5",
  111. )
  112. auto_http_archive(
  113. name = "com_github_tporadowski_redis_bin",
  114. build_file = "@com_github_ray_project_ray//bazel:BUILD.redis",
  115. strip_prefix = None,
  116. url = "https://github.com/tporadowski/redis/releases/download/v5.0.9/Redis-x64-5.0.9.zip",
  117. sha256 = "b09565b22b50c505a5faa86a7e40b6683afb22f3c17c5e6a5e35fc9b7c03f4c2",
  118. )
  119. auto_http_archive(
  120. name = "rules_jvm_external",
  121. url = "https://github.com/bazelbuild/rules_jvm_external/archive/2.10.tar.gz",
  122. sha256 = "5c1b22eab26807d5286ada7392d796cbc8425d3ef9a57d114b79c5f8ef8aca7c",
  123. )
  124. auto_http_archive(
  125. name = "bazel_common",
  126. url = "https://github.com/google/bazel-common/archive/084aadd3b854cad5d5e754a7e7d958ac531e6801.tar.gz",
  127. sha256 = "a6e372118bc961b182a3a86344c0385b6b509882929c6b12dc03bb5084c775d5",
  128. )
  129. auto_http_archive(
  130. name = "bazel_skylib",
  131. strip_prefix = None,
  132. url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
  133. sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
  134. )
  135. auto_http_archive(
  136. # This rule is used by @com_github_nelhage_rules_boost and
  137. # declaring it here allows us to avoid patching the latter.
  138. name = "boost",
  139. build_file = "@com_github_nelhage_rules_boost//:BUILD.boost",
  140. sha256 = "83bfc1507731a0906e387fc28b7ef5417d591429e51e788417fe9ff025e116b1",
  141. url = "https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source/boost_1_74_0.tar.bz2",
  142. patches = [
  143. "@com_github_ray_project_ray//thirdparty/patches:boost-exception-no_warn_typeid_evaluated.patch",
  144. ],
  145. )
  146. auto_http_archive(
  147. name = "com_github_nelhage_rules_boost",
  148. # If you update the Boost version, remember to update the 'boost' rule.
  149. url = "https://github.com/nelhage/rules_boost/archive/652b21e35e4eeed5579e696da0facbe8dba52b1f.tar.gz",
  150. sha256 = "c1b8b2adc3b4201683cf94dda7eef3fc0f4f4c0ea5caa3ed3feffe07e1fb5b15",
  151. patches = [
  152. "@com_github_ray_project_ray//thirdparty/patches:rules_boost-windows-linkopts.patch",
  153. ],
  154. )
  155. auto_http_archive(
  156. name = "com_github_google_flatbuffers",
  157. url = "https://github.com/google/flatbuffers/archive/63d51afd1196336a7d1f56a988091ef05deb1c62.tar.gz",
  158. sha256 = "3f469032571d324eabea88d7014c05fec8565a5877dbe49b2a52d8d1a0f18e63",
  159. )
  160. auto_http_archive(
  161. name = "com_google_googletest",
  162. url = "https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz",
  163. sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
  164. )
  165. auto_http_archive(
  166. name = "com_github_gflags_gflags",
  167. url = "https://github.com/gflags/gflags/archive/e171aa2d15ed9eb17054558e0b3a6a413bb01067.tar.gz",
  168. sha256 = "b20f58e7f210ceb0e768eb1476073d0748af9b19dfbbf53f4fd16e3fb49c5ac8",
  169. )
  170. auto_http_archive(
  171. name = "cython",
  172. build_file = True,
  173. url = "https://github.com/cython/cython/archive/3028e8c7ac296bc848d996e397c3354b3dbbd431.tar.gz",
  174. sha256 = "31ea23c2231ddee8572a2a5effd54952e16a1b44e9a4cb3eb645418f8accf20d",
  175. )
  176. auto_http_archive(
  177. name = "com_github_johnynek_bazel_jar_jar",
  178. url = "https://github.com/johnynek/bazel_jar_jar/archive/171f268569384c57c19474b04aebe574d85fde0d.tar.gz",
  179. sha256 = "97c5f862482a05f385bd8f9d28a9bbf684b0cf3fae93112ee96f3fb04d34b193",
  180. )
  181. auto_http_archive(
  182. name = "io_opencensus_cpp",
  183. url = "https://github.com/census-instrumentation/opencensus-cpp/archive/b14a5c0dcc2da8a7fc438fab637845c73438b703.zip",
  184. sha256 = "6592e07672e7f7980687f6c1abda81974d8d379e273fea3b54b6c4d855489b9d",
  185. patches = [
  186. "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-harvest-interval.patch",
  187. "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-shutdown-api.patch",
  188. ],
  189. )
  190. # OpenCensus depends on Abseil so we have to explicitly pull it in.
  191. # This is how diamond dependencies are prevented.
  192. auto_http_archive(
  193. name = "com_google_absl",
  194. url = "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz",
  195. sha256 = "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4",
  196. )
  197. # OpenCensus depends on jupp0r/prometheus-cpp
  198. auto_http_archive(
  199. name = "com_github_jupp0r_prometheus_cpp",
  200. url = "https://github.com/jupp0r/prometheus-cpp/archive/60eaa4ea47b16751a8e8740b05fe70914c68a480.tar.gz",
  201. sha256 = "ec825b802487ac18b0d98e2e8b7961487b12562f8f82e424521d0a891d9e1373",
  202. patches = [
  203. "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-headers.patch",
  204. # https://github.com/jupp0r/prometheus-cpp/pull/225
  205. "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-zlib.patch",
  206. "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-pollfd.patch",
  207. ],
  208. )
  209. auto_http_archive(
  210. name = "com_github_grpc_grpc",
  211. # NOTE: If you update this, also update @boringssl's hash.
  212. url = "https://github.com/grpc/grpc/archive/refs/tags/v1.42.0.tar.gz",
  213. sha256 = "b2f2620c762427bfeeef96a68c1924319f384e877bc0e084487601e4cc6e434c",
  214. patches = [
  215. "@com_github_ray_project_ray//thirdparty/patches:grpc-cython-copts.patch",
  216. # Delete after upgrading from 1.42.0
  217. "@com_github_ray_project_ray//thirdparty/patches:grpc-default-initialization.patch",
  218. "@com_github_ray_project_ray//thirdparty/patches:grpc-python.patch",
  219. ],
  220. )
  221. http_archive(
  222. # This rule is used by @com_github_grpc_grpc, and using a GitHub mirror
  223. # provides a deterministic archive hash for caching. Explanation here:
  224. # https://github.com/grpc/grpc/blob/1ff1feaa83e071d87c07827b0a317ffac673794f/bazel/grpc_deps.bzl#L189
  225. # Ensure this rule matches the rule used by grpc's bazel/grpc_deps.bzl
  226. name = "boringssl",
  227. sha256 = "e168777eb0fc14ea5a65749a2f53c095935a6ea65f38899a289808fb0c221dc4",
  228. strip_prefix = "boringssl-4fb158925f7753d80fb858cb0239dff893ef9f15",
  229. urls = [
  230. "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/4fb158925f7753d80fb858cb0239dff893ef9f15.tar.gz",
  231. "https://github.com/google/boringssl/archive/4fb158925f7753d80fb858cb0239dff893ef9f15.tar.gz",
  232. ],
  233. )
  234. auto_http_archive(
  235. name = "rules_proto_grpc",
  236. url = "https://github.com/rules-proto-grpc/rules_proto_grpc/archive/a74fef39c5fe636580083545f76d1eab74f6450d.tar.gz",
  237. sha256 = "2f6606151ec042e23396f07de9e7dcf6ca9a5db1d2b09f0cc93a7fc7f4008d1b",
  238. )
  239. auto_http_archive(
  240. name = "msgpack",
  241. build_file = True,
  242. url = "https://github.com/msgpack/msgpack-c/archive/8085ab8721090a447cf98bb802d1406ad7afe420.tar.gz",
  243. sha256 = "83c37c9ad926bbee68d564d9f53c6cbb057c1f755c264043ddd87d89e36d15bb",
  244. patches = [
  245. "@com_github_ray_project_ray//thirdparty/patches:msgpack-windows-iovec.patch",
  246. ],
  247. )
  248. http_archive(
  249. name = "io_opencensus_proto",
  250. strip_prefix = "opencensus-proto-0.3.0/src",
  251. urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz"],
  252. sha256 = "b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0",
  253. )
  254. http_archive(
  255. name = "nlohmann_json",
  256. strip_prefix = "json-3.9.1",
  257. urls = ["https://github.com/nlohmann/json/archive/v3.9.1.tar.gz"],
  258. sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b",
  259. build_file = "@com_github_ray_project_ray//bazel:BUILD.nlohmann_json",
  260. )
  261. auto_http_archive(
  262. name = "rapidjson",
  263. url = "https://github.com/Tencent/rapidjson/archive/v1.1.0.zip",
  264. build_file = True,
  265. sha256 = "8e00c38829d6785a2dfb951bb87c6974fa07dfe488aa5b25deec4b8bc0f6a3ab",
  266. )