BUILD.bazel 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. load("//bazel:ray.bzl", "define_java_module")
  2. load("//bazel:ray.bzl", "native_java_binary")
  3. load("//bazel:ray.bzl", "native_java_library")
  4. load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
  5. load(
  6. "@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl",
  7. "jar_jar",
  8. )
  9. exports_files([
  10. "testng.xml",
  11. "checkstyle.xml",
  12. "checkstyle-suppressions.xml",
  13. ])
  14. all_modules = [
  15. "api",
  16. "runtime",
  17. "serve",
  18. "test",
  19. "performance_test",
  20. ]
  21. java_import(
  22. name = "all_modules",
  23. jars = [
  24. "libio_ray_ray_" + module + ".jar"
  25. for module in all_modules
  26. ] + [
  27. "libio_ray_ray_" + module + "-src.jar"
  28. for module in all_modules
  29. ] + [
  30. "all_tests_deploy.jar",
  31. "all_tests_deploy-src.jar",
  32. ],
  33. deps = [
  34. ":io_ray_ray_" + module
  35. for module in all_modules
  36. ] + [
  37. ":all_tests",
  38. ],
  39. )
  40. define_java_module(
  41. name = "api",
  42. visibility = ["//visibility:public"],
  43. deps = [
  44. "@maven//:com_lmax_disruptor",
  45. "@maven//:com_sun_xml_bind_jaxb_core",
  46. "@maven//:com_sun_xml_bind_jaxb_impl",
  47. "@maven//:javax_activation_activation",
  48. "@maven//:javax_xml_bind_jaxb_api",
  49. "@maven//:org_apache_logging_log4j_log4j_api",
  50. "@maven//:org_apache_logging_log4j_log4j_core",
  51. "@maven//:org_apache_logging_log4j_log4j_slf4j_impl",
  52. "@maven//:org_slf4j_slf4j_api",
  53. ],
  54. )
  55. define_java_module(
  56. name = "runtime",
  57. additional_resources = [
  58. ":java_native_deps",
  59. ],
  60. additional_srcs = [
  61. ":all_java_proto",
  62. ],
  63. define_test_lib = True,
  64. exclude_srcs = [
  65. "runtime/src/main/java/io/ray/runtime/generated/*.java",
  66. ],
  67. test_deps = [
  68. ":io_ray_ray_api",
  69. ":io_ray_ray_runtime",
  70. "@maven//:commons_io_commons_io",
  71. "@maven//:javax_xml_bind_jaxb_api",
  72. "@maven//:org_apache_commons_commons_lang3",
  73. "@maven//:org_testng_testng",
  74. ],
  75. visibility = ["//visibility:public"],
  76. deps = [
  77. ":io_ray_ray_api",
  78. "@maven//:com_fasterxml_jackson_core_jackson_databind",
  79. "@maven//:com_github_java_json_tools_json_schema_validator",
  80. "@maven//:com_google_code_gson_gson",
  81. "@maven//:com_google_guava_guava",
  82. "@maven//:com_google_protobuf_protobuf_java",
  83. "@maven//:com_google_protobuf_protobuf_java_util",
  84. "@maven//:com_lmax_disruptor",
  85. "@maven//:com_typesafe_config",
  86. "@maven//:commons_io_commons_io",
  87. "@maven//:de_ruedigermoeller_fst",
  88. "@maven//:net_java_dev_jna_jna",
  89. "@maven//:org_apache_commons_commons_lang3",
  90. "@maven//:org_apache_logging_log4j_log4j_api",
  91. "@maven//:org_apache_logging_log4j_log4j_core",
  92. "@maven//:org_apache_logging_log4j_log4j_slf4j_impl",
  93. "@maven//:org_msgpack_msgpack_core",
  94. "@maven//:org_ow2_asm_asm",
  95. "@maven//:org_slf4j_slf4j_api",
  96. "@maven//:org_testng_testng",
  97. ],
  98. )
  99. define_java_module(
  100. name = "test",
  101. # (WangTaoTheTonic)For cpp x-lang tests. See //cpp:cluster_mode_xlang_test.
  102. visibility = ["//cpp:__subpackages__"],
  103. deps = [
  104. ":io_ray_ray_api",
  105. ":io_ray_ray_runtime",
  106. "@maven//:com_google_code_gson_gson",
  107. "@maven//:com_google_guava_guava",
  108. "@maven//:com_google_protobuf_protobuf_java",
  109. "@maven//:com_lmax_disruptor",
  110. "@maven//:com_sun_xml_bind_jaxb_core",
  111. "@maven//:com_sun_xml_bind_jaxb_impl",
  112. "@maven//:commons_io_commons_io",
  113. "@maven//:javax_xml_bind_jaxb_api",
  114. "@maven//:org_apache_commons_commons_lang3",
  115. "@maven//:org_apache_logging_log4j_log4j_api",
  116. "@maven//:org_apache_logging_log4j_log4j_core",
  117. "@maven//:org_apache_logging_log4j_log4j_slf4j_impl",
  118. "@maven//:org_slf4j_slf4j_api",
  119. "@maven//:org_testng_testng",
  120. ],
  121. )
  122. define_java_module(
  123. name = "performance_test",
  124. deps = [
  125. ":io_ray_ray_api",
  126. ":io_ray_ray_runtime",
  127. "@maven//:com_google_code_gson_gson",
  128. "@maven//:com_google_guava_guava",
  129. "@maven//:com_lmax_disruptor",
  130. "@maven//:commons_io_commons_io",
  131. "@maven//:org_apache_commons_commons_lang3",
  132. "@maven//:org_apache_logging_log4j_log4j_api",
  133. "@maven//:org_apache_logging_log4j_log4j_core",
  134. "@maven//:org_apache_logging_log4j_log4j_slf4j_impl",
  135. "@maven//:org_slf4j_slf4j_api",
  136. ],
  137. )
  138. define_java_module(
  139. name = "serve",
  140. additional_srcs = [
  141. ":serve_java_proto",
  142. ],
  143. define_test_lib = True,
  144. exclude_srcs = [
  145. "serve/src/main/java/io/ray/serve/generated/*.java",
  146. ],
  147. test_deps = [
  148. ":io_ray_ray_api",
  149. ":io_ray_ray_runtime",
  150. ":io_ray_ray_serve",
  151. "@maven//:com_google_code_gson_gson",
  152. "@maven//:com_google_guava_guava",
  153. "@maven//:com_google_protobuf_protobuf_java",
  154. "@maven//:commons_io_commons_io",
  155. "@maven//:org_apache_commons_commons_lang3",
  156. "@maven//:org_apache_httpcomponents_client5_httpclient5",
  157. "@maven//:org_apache_httpcomponents_client5_httpclient5_fluent",
  158. "@maven//:org_apache_httpcomponents_core5_httpcore5",
  159. "@maven//:org_slf4j_slf4j_api",
  160. "@maven//:org_testng_testng",
  161. ],
  162. visibility = ["//visibility:public"],
  163. deps = [
  164. ":io_ray_ray_api",
  165. ":io_ray_ray_runtime",
  166. "@maven//:com_google_code_gson_gson",
  167. "@maven//:com_google_guava_guava",
  168. "@maven//:com_google_protobuf_protobuf_java",
  169. "@maven//:org_apache_commons_commons_lang3",
  170. "@maven//:org_apache_httpcomponents_core5_httpcore5",
  171. "@maven//:org_slf4j_slf4j_api",
  172. ],
  173. )
  174. java_library(
  175. name = "all_tests_lib",
  176. runtime_deps = [
  177. ":io_ray_ray_performance_test",
  178. ":io_ray_ray_runtime_test",
  179. ":io_ray_ray_serve_test",
  180. ":io_ray_ray_test",
  181. ],
  182. )
  183. java_test(
  184. name = "all_tests",
  185. testonly = True,
  186. args = ["java/testng.xml"],
  187. data = [
  188. "testng.xml",
  189. ":ray_java_pkg",
  190. "//:ray_pkg",
  191. ],
  192. main_class = "org.testng.TestNG",
  193. resources = [
  194. "//cpp:counter.so",
  195. "//cpp:plus.so",
  196. ],
  197. tags = ["local"],
  198. runtime_deps = [
  199. ":all_tests_lib",
  200. ],
  201. )
  202. # 0. `cp testng_custom_template.xml testng_custom.xml`
  203. # 1. Specify test class/method in `testng_custom.xml`
  204. # 2. `bazel test //java:custom_test --test_output=streamed`
  205. java_test(
  206. name = "custom_test",
  207. args = ["java/testng_custom.xml"],
  208. data = [
  209. "testng_custom.xml",
  210. ":ray_java_pkg",
  211. "//:ray_pkg",
  212. ],
  213. main_class = "org.testng.TestNG",
  214. tags = ["local"],
  215. runtime_deps = [
  216. ":all_tests_lib",
  217. ],
  218. )
  219. # We'd better make resource files can be accessed from 3rd party library.
  220. # More detail please see https://github.com/ray-project/ray/pull/21641.
  221. java_proto_compile(
  222. name = "common_java_proto",
  223. deps = ["@com_github_ray_project_ray//src/ray/protobuf:common_proto"],
  224. )
  225. java_proto_compile(
  226. name = "runtime_env_common_java_proto",
  227. deps = ["@com_github_ray_project_ray//src/ray/protobuf:runtime_env_common_proto"],
  228. )
  229. java_proto_compile(
  230. name = "gcs_java_proto",
  231. deps = ["@com_github_ray_project_ray//src/ray/protobuf:gcs_proto"],
  232. )
  233. java_proto_compile(
  234. name = "serve_java_proto",
  235. deps = ["@com_github_ray_project_ray//src/ray/protobuf:serve_proto"],
  236. )
  237. filegroup(
  238. name = "all_java_proto",
  239. srcs = [
  240. ":common_java_proto",
  241. ":gcs_java_proto",
  242. ":runtime_env_common_java_proto",
  243. ],
  244. )
  245. native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
  246. filegroup(
  247. name = "java_native_deps",
  248. srcs = [
  249. ":core_worker_library_java",
  250. ],
  251. )
  252. # Generates the dependencies needed by maven.
  253. genrule(
  254. name = "cp_java_generated",
  255. srcs = [
  256. ":all_java_proto",
  257. ":copy_pom_file",
  258. ":serve_java_proto",
  259. ],
  260. outs = ["cp_java_generated.out"],
  261. cmd = """
  262. WORK_DIR="$$(pwd)"
  263. # Copy protobuf-generated files.
  264. rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated"
  265. for f in $(locations //java:all_java_proto); do
  266. unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
  267. done
  268. rm -rf "$$WORK_DIR/java/serve/src/main/java/io/ray/serve/generated"
  269. for f in $(locations //java:serve_java_proto); do
  270. unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/serve/src/main/java"
  271. done
  272. date > $@
  273. """,
  274. local = 1,
  275. tags = ["no-cache"],
  276. )
  277. # Generates the dependencies needed by maven.
  278. genrule(
  279. name = "gen_maven_deps",
  280. srcs = [
  281. ":cp_java_generated",
  282. ":java_native_deps",
  283. ],
  284. outs = ["gen_maven_deps.out"],
  285. cmd = """
  286. WORK_DIR="$${PWD}"
  287. # Copy native dependencies.
  288. OS_NAME=""
  289. case "$${OSTYPE}" in
  290. linux*) OS_NAME="linux";;
  291. darwin*) OS_NAME="darwin";;
  292. *) echo "$${OSTYPE} is not supported currently"; exit 1;;
  293. esac
  294. NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
  295. rm -rf "$$NATIVE_DEPS_DIR"
  296. mkdir -p "$$NATIVE_DEPS_DIR"
  297. for f in $(locations //java:java_native_deps); do
  298. chmod +w "$$f"
  299. cp "$$f" "$$NATIVE_DEPS_DIR"
  300. done
  301. date > $@
  302. """,
  303. local = 1,
  304. tags = ["no-cache"],
  305. )
  306. genrule(
  307. name = "copy_pom_file",
  308. srcs = [
  309. "//java:io_ray_ray_" + module + "_pom"
  310. for module in all_modules
  311. ],
  312. outs = ["copy_pom_file.out"],
  313. cmd = """
  314. WORK_DIR="$$(pwd)"
  315. cp -f $(location //java:io_ray_ray_api_pom) "$$WORK_DIR/java/api/pom.xml"
  316. cp -f $(location //java:io_ray_ray_runtime_pom) "$$WORK_DIR/java/runtime/pom.xml"
  317. cp -f $(location //java:io_ray_ray_test_pom) "$$WORK_DIR/java/test/pom.xml"
  318. cp -f $(location //java:io_ray_ray_performance_test_pom) "$$WORK_DIR/java/performance_test/pom.xml"
  319. cp -f $(location //java:io_ray_ray_serve_pom) "$$WORK_DIR/java/serve/pom.xml"
  320. date > $@
  321. """,
  322. local = 1,
  323. tags = ["no-cache"],
  324. )
  325. java_binary(
  326. name = "ray_dist",
  327. # This rule is used to package all Ray Java code and the third-party dependencies into a
  328. # fat jar file. It's not really an executable jar. So we set its `main_class` to empty.
  329. main_class = "",
  330. runtime_deps = [
  331. "//java:io_ray_ray_api",
  332. "//java:io_ray_ray_runtime",
  333. "//java:io_ray_ray_serve",
  334. ],
  335. )
  336. jar_jar(
  337. name = "ray_dist_shaded",
  338. input_jar = "//java:ray_dist_deploy.jar",
  339. rules = "//java:shade_rule",
  340. )
  341. # Shade dependencies in tests fat jar.
  342. jar_jar(
  343. name = "all_tests_shaded",
  344. input_jar = "//java:all_tests_deploy.jar",
  345. rules = "//java:shade_rule",
  346. )
  347. genrule(
  348. name = "ray_java_pkg",
  349. srcs = [
  350. "//java:ray_dist_shaded.jar",
  351. "//java:gen_maven_deps",
  352. ],
  353. outs = ["ray_java_pkg.out"],
  354. cmd = """
  355. WORK_DIR="$$(pwd)"
  356. rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars"
  357. cp -f $(location //java:ray_dist_shaded.jar) "$$WORK_DIR/python/ray/jars/ray_dist.jar"
  358. date > $@
  359. """,
  360. local = 1,
  361. tags = ["no-cache"],
  362. )