build-jar-multiplatform.sh 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. #!/bin/bash
  2. set -x
  3. # Cause the script to exit if a single command fails.
  4. set -e
  5. ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
  6. WORKSPACE_DIR="${ROOT_DIR}/.."
  7. JAVA_DIRS_PATH=('java' 'streaming/java')
  8. RAY_JAVA_MODULES=('api' 'runtime')
  9. RAY_STREAMING_JAVA_MODULES=('streaming-api' 'streaming-runtime' 'streaming-state')
  10. JAR_BASE_DIR="$WORKSPACE_DIR"/.jar
  11. mkdir -p "$JAR_BASE_DIR"
  12. cd "$WORKSPACE_DIR/java"
  13. # ray jar version, ex: 0.1-SNAPSHORT
  14. version=$(python -c "import xml.etree.ElementTree as ET; r = ET.parse('pom.xml').getroot(); print(r.find(r.tag.replace('project', 'version')).text);" | tail -n 1)
  15. cd -
  16. check_java_version() {
  17. local VERSION
  18. VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
  19. if [[ ! $VERSION =~ 1.8 ]]; then
  20. echo "Java version is $VERSION. Please install jkd8."
  21. exit 1
  22. fi
  23. }
  24. build_jars() {
  25. local platform="$1"
  26. local bazel_build="${2:-true}"
  27. echo "bazel_build $bazel_build"
  28. echo "Start building jar for $platform"
  29. local JAR_DIR="$JAR_BASE_DIR/$platform"
  30. mkdir -p "$JAR_DIR"
  31. for p in "${JAVA_DIRS_PATH[@]}"; do
  32. cd "$WORKSPACE_DIR/$p"
  33. bazel build cp_java_generated
  34. if [[ $bazel_build == "true" ]]; then
  35. echo "Starting building java native dependencies for $p"
  36. bazel build gen_maven_deps
  37. echo "Finished building java native dependencies for $p"
  38. fi
  39. echo "Start building jars for $p"
  40. mvn -T16 clean package install -Dmaven.test.skip=true -Dcheckstyle.skip
  41. mvn -T16 source:jar -Dmaven.test.skip=true -Dcheckstyle.skip
  42. echo "Finished building jars for $p"
  43. done
  44. copy_jars "$JAR_DIR"
  45. # ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
  46. cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
  47. cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
  48. echo "Finished building jar for $platform"
  49. }
  50. copy_jars() {
  51. local JAR_DIR="$1"
  52. echo "Copy to dir $JAR_DIR"
  53. for module in "${RAY_JAVA_MODULES[@]}"; do
  54. cp -f "$WORKSPACE_DIR"/java/"$module"/target/*jar "$JAR_DIR"
  55. done
  56. for module in "${RAY_STREAMING_JAVA_MODULES[@]}"; do
  57. cp -f "$WORKSPACE_DIR"/streaming/java/"$module"/target/*jar "$JAR_DIR"
  58. done
  59. # ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
  60. cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
  61. cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
  62. }
  63. # This function assuem all dependencies are installed already.
  64. build_jars_linux() {
  65. build_jars linux
  66. }
  67. # This function assuem all dependencies are installed already.
  68. build_jars_darwin() {
  69. build_jars darwin
  70. }
  71. build_jars_multiplatform() {
  72. if [ "${TRAVIS-}" = true ]; then
  73. if [[ "${TRAVIS_REPO_SLUG-}" != "ray-project/ray" || "${TRAVIS_PULL_REQUEST-}" != "false" ]]; then
  74. echo "Skip build multiplatform jars when this build is from a pull request or
  75. not a build for commit in ray-project/ray."
  76. return
  77. fi
  78. fi
  79. if download_jars "ray-runtime-$version.jar" "streaming-runtime-$version.jar"; then
  80. prepare_native
  81. build_jars multiplatform false
  82. else
  83. echo "download_jars failed, skip building multiplatform jars"
  84. fi
  85. }
  86. # Download darwin/windows ray-related jar from s3
  87. # This function assumes linux jars exist already.
  88. download_jars() {
  89. local wait_time=0
  90. local sleep_time_units=60
  91. for f in "$@"; do
  92. for os in 'darwin' 'linux' 'windows'; do
  93. if [[ "$os" == "windows" ]]; then
  94. continue
  95. fi
  96. local url="https://ray-wheels.s3-us-west-2.amazonaws.com/jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT/$os/$f"
  97. mkdir -p "$JAR_BASE_DIR/$os"
  98. local dest_file="$JAR_BASE_DIR/$os/$f"
  99. echo "Jar url: $url"
  100. echo "Jar dest_file: $dest_file"
  101. while true; do
  102. if ! wget -q "$url" -O "$dest_file">/dev/null; then
  103. echo "Waiting $url to be ready for $wait_time seconds..."
  104. sleep $sleep_time_units
  105. wait_time=$((wait_time + sleep_time_units))
  106. if [[ wait_time == $((sleep_time_units * 100)) ]]; then
  107. echo "Download $url timeout"
  108. return 1
  109. fi
  110. else
  111. echo "Download $url to $dest_file succeed"
  112. break
  113. fi
  114. done
  115. done
  116. done
  117. echo "Download jars took $wait_time seconds"
  118. }
  119. # prepare native binaries and libraries.
  120. prepare_native() {
  121. for os in 'darwin' 'linux'; do
  122. cd "$JAR_BASE_DIR/$os"
  123. jar xf "ray-runtime-$version.jar" "native/$os"
  124. local native_dir="$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os"
  125. mkdir -p "$native_dir"
  126. rm -rf "$native_dir"
  127. mv "native/$os" "$native_dir"
  128. jar xf "streaming-runtime-$version.jar" "native/$os"
  129. local native_dir="$WORKSPACE_DIR/streaming/java/streaming-runtime/native_dependencies/native/$os"
  130. mkdir -p "$native_dir"
  131. rm -rf "$native_dir"
  132. mv "native/$os" "$native_dir"
  133. done
  134. }
  135. # Return 0 if native bianries and libraries exist and 1 if not.
  136. native_files_exist() {
  137. local os
  138. for os in 'darwin' 'linux'; do
  139. native_dirs=()
  140. native_dirs+=("$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os")
  141. native_dirs+=("$WORKSPACE_DIR/streaming/java/streaming-runtime/native_dependencies/native/$os")
  142. for native_dir in "${native_dirs[@]}"; do
  143. if [ ! -d "$native_dir" ]; then
  144. echo "$native_dir doesn't exist"
  145. return 1
  146. fi
  147. done
  148. done
  149. }
  150. # This function assume all multiplatform binaries are prepared already.
  151. deploy_jars() {
  152. if [ "${TRAVIS-}" = true ]; then
  153. mkdir -p ~/.m2
  154. echo "<settings><servers><server><id>ossrh</id><username>${OSSRH_KEY}</username><password>${OSSRH_TOKEN}</password></server></servers></settings>" > ~/.m2/settings.xml
  155. if [[ "$TRAVIS_REPO_SLUG" != "ray-project/ray" ||
  156. "$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_BRANCH" != "master" ]]; then
  157. echo "Skip deploying jars when this build is from a pull request or
  158. not a build for commit of master branch in ray-project/ray"
  159. return
  160. fi
  161. fi
  162. echo "Start deploying jars"
  163. if native_files_exist; then
  164. (
  165. cd "$WORKSPACE_DIR/java"
  166. mvn -T16 install deploy -Dmaven.test.skip=true -Dcheckstyle.skip -Prelease -Dgpg.skip="${GPG_SKIP:-true}"
  167. )
  168. (
  169. cd "$WORKSPACE_DIR/streaming/java"
  170. mvn -T16 deploy -Dmaven.test.skip=true -Dcheckstyle.skip -Prelease -Dgpg.skip="${GPG_SKIP:-true}"
  171. )
  172. echo "Finished deploying jars"
  173. else
  174. echo "Native bianries/libraries are not ready, skip deploying jars."
  175. fi
  176. }
  177. if [ -z "${BUILDKITE-}" ]; then
  178. check_java_version
  179. fi
  180. case "$1" in
  181. linux) # build jars that only contains Linux binaries.
  182. build_jars_linux
  183. ;;
  184. darwin) # build jars that only contains macos binaries.
  185. build_jars_darwin
  186. ;;
  187. multiplatform) # downloading jars of multiple platforms and packaging them into one jar.
  188. build_jars_multiplatform
  189. ;;
  190. deploy) # Deploy jars to maven repository.
  191. deploy_jars
  192. ;;
  193. *)
  194. echo "Execute command $*"
  195. "$@"
  196. ;;
  197. esac