build-jar-multiplatform.sh 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. #!/bin/bash
  2. set -x
  3. # Cause the script to exit if a single command fails.
  4. set -e
  5. ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
  6. WORKSPACE_DIR="${ROOT_DIR}/.."
  7. JAVA_DIRS_PATH=('java')
  8. RAY_JAVA_MODULES=('api' 'runtime' 'serve')
  9. JAR_BASE_DIR="$WORKSPACE_DIR"/.jar
  10. mkdir -p "$JAR_BASE_DIR"
  11. cd "$WORKSPACE_DIR/java"
  12. # ray jar version, ex: 0.1-SNAPSHORT
  13. version=$(python -c "import xml.etree.ElementTree as ET; r = ET.parse('pom.xml').getroot(); print(r.find(r.tag.replace('project', 'version')).text);" | tail -n 1)
  14. cd -
  15. check_java_version() {
  16. local VERSION
  17. VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
  18. if [[ ! $VERSION =~ 1.8 ]]; then
  19. echo "Java version is $VERSION. Please install jkd8."
  20. exit 1
  21. fi
  22. }
  23. build_jars() {
  24. local platform="$1"
  25. local bazel_build="${2:-true}"
  26. echo "bazel_build $bazel_build"
  27. echo "Start building jar for $platform"
  28. local JAR_DIR="$JAR_BASE_DIR/$platform"
  29. mkdir -p "$JAR_DIR"
  30. for p in "${JAVA_DIRS_PATH[@]}"; do
  31. cd "$WORKSPACE_DIR/$p"
  32. bazel build cp_java_generated
  33. if [[ $bazel_build == "true" ]]; then
  34. echo "Starting building java native dependencies for $p"
  35. bazel build gen_maven_deps
  36. echo "Finished building java native dependencies for $p"
  37. fi
  38. echo "Start building jars for $p"
  39. mvn -T16 clean package install -Dmaven.test.skip=true -Dcheckstyle.skip
  40. mvn -T16 source:jar -Dmaven.test.skip=true -Dcheckstyle.skip
  41. echo "Finished building jars for $p"
  42. done
  43. copy_jars "$JAR_DIR"
  44. # ray runtime jar is in a dir specifed by maven-jar-plugin
  45. cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
  46. echo "Finished building jar for $platform"
  47. }
  48. copy_jars() {
  49. local JAR_DIR="$1"
  50. echo "Copy to dir $JAR_DIR"
  51. for module in "${RAY_JAVA_MODULES[@]}"; do
  52. cp -f "$WORKSPACE_DIR"/java/"$module"/target/*jar "$JAR_DIR"
  53. done
  54. # ray runtime jar is in a dir specifed by maven-jar-plugin
  55. cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
  56. }
  57. # This function assuem all dependencies are installed already.
  58. build_jars_linux() {
  59. build_jars linux
  60. }
  61. # This function assuem all dependencies are installed already.
  62. build_jars_darwin() {
  63. build_jars darwin
  64. }
  65. build_jars_multiplatform() {
  66. if [ "${TRAVIS-}" = true ]; then
  67. if [[ "${TRAVIS_REPO_SLUG-}" != "ray-project/ray" || "${TRAVIS_PULL_REQUEST-}" != "false" ]]; then
  68. echo "Skip build multiplatform jars when this build is from a pull request or
  69. not a build for commit in ray-project/ray."
  70. return
  71. fi
  72. fi
  73. if download_jars "ray-runtime-$version.jar"; then
  74. prepare_native
  75. build_jars multiplatform false
  76. else
  77. echo "download_jars failed, skip building multiplatform jars"
  78. fi
  79. }
  80. # Download darwin/windows ray-related jar from s3
  81. # This function assumes linux jars exist already.
  82. download_jars() {
  83. local wait_time=0
  84. local sleep_time_units=60
  85. for f in "$@"; do
  86. for os in 'darwin' 'linux' 'windows'; do
  87. if [[ "$os" == "windows" ]]; then
  88. continue
  89. fi
  90. local url="https://ray-wheels.s3-us-west-2.amazonaws.com/jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT/$os/$f"
  91. mkdir -p "$JAR_BASE_DIR/$os"
  92. local dest_file="$JAR_BASE_DIR/$os/$f"
  93. echo "Jar url: $url"
  94. echo "Jar dest_file: $dest_file"
  95. while true; do
  96. if ! wget -q "$url" -O "$dest_file">/dev/null; then
  97. echo "Waiting $url to be ready for $wait_time seconds..."
  98. sleep $sleep_time_units
  99. wait_time=$((wait_time + sleep_time_units))
  100. if [[ wait_time == $((sleep_time_units * 100)) ]]; then
  101. echo "Download $url timeout"
  102. return 1
  103. fi
  104. else
  105. echo "Download $url to $dest_file succeed"
  106. break
  107. fi
  108. done
  109. done
  110. done
  111. echo "Download jars took $wait_time seconds"
  112. }
  113. # prepare native binaries and libraries.
  114. prepare_native() {
  115. for os in 'darwin' 'linux'; do
  116. cd "$JAR_BASE_DIR/$os"
  117. jar xf "ray-runtime-$version.jar" "native/$os"
  118. local native_dir="$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os"
  119. mkdir -p "$native_dir"
  120. rm -rf "$native_dir"
  121. mv "native/$os" "$native_dir"
  122. done
  123. }
  124. # Return 0 if native bianries and libraries exist and 1 if not.
  125. native_files_exist() {
  126. local os
  127. for os in 'darwin' 'linux'; do
  128. native_dirs=()
  129. native_dirs+=("$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os")
  130. for native_dir in "${native_dirs[@]}"; do
  131. if [ ! -d "$native_dir" ]; then
  132. echo "$native_dir doesn't exist"
  133. return 1
  134. fi
  135. done
  136. done
  137. }
  138. # This function assume all multiplatform binaries are prepared already.
  139. deploy_jars() {
  140. if [ "${TRAVIS-}" = true ]; then
  141. mkdir -p ~/.m2
  142. echo "<settings><servers><server><id>ossrh</id><username>${OSSRH_KEY}</username><password>${OSSRH_TOKEN}</password></server></servers></settings>" > ~/.m2/settings.xml
  143. if [[ "$TRAVIS_REPO_SLUG" != "ray-project/ray" ||
  144. "$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_BRANCH" != "master" ]]; then
  145. echo "Skip deploying jars when this build is from a pull request or
  146. not a build for commit of master branch in ray-project/ray"
  147. return
  148. fi
  149. fi
  150. echo "Start deploying jars"
  151. if native_files_exist; then
  152. (
  153. cd "$WORKSPACE_DIR/java"
  154. mvn -T16 install deploy -Dmaven.test.skip=true -Dcheckstyle.skip -Prelease -Dgpg.skip="${GPG_SKIP:-true}"
  155. )
  156. echo "Finished deploying jars"
  157. else
  158. echo "Native bianries/libraries are not ready, skip deploying jars."
  159. fi
  160. }
  161. if [ -z "${BUILDKITE-}" ]; then
  162. check_java_version
  163. fi
  164. case "$1" in
  165. linux) # build jars that only contains Linux binaries.
  166. build_jars_linux
  167. ;;
  168. darwin) # build jars that only contains macos binaries.
  169. build_jars_darwin
  170. ;;
  171. multiplatform) # downloading jars of multiple platforms and packaging them into one jar.
  172. build_jars_multiplatform
  173. ;;
  174. deploy) # Deploy jars to maven repository.
  175. deploy_jars
  176. ;;
  177. *)
  178. echo "Execute command $*"
  179. "$@"
  180. ;;
  181. esac