build-wheel-manylinux2014.sh 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. #!/bin/bash
  2. set -x
  3. # Cause the script to exit if a single command fails.
  4. set -euo pipefail
  5. cat << EOF > "/usr/bin/nproc"
  6. #!/bin/bash
  7. echo 10
  8. EOF
  9. chmod +x /usr/bin/nproc
  10. NODE_VERSION="14"
  11. # Python version key, interpreter version code, numpy tuples.
  12. PYTHON_NUMPYS=(
  13. "3.7 cp37-cp37m 1.14.5"
  14. "3.8 cp38-cp38 1.14.5"
  15. "3.9 cp39-cp39 1.19.3"
  16. "3.10 cp310-cp310 1.22.0"
  17. "3.11 cp311-cp311 1.22.0"
  18. )
  19. yum -y install unzip zip sudo
  20. yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz
  21. yum -y install openssl
  22. if [[ "${HOSTTYPE-}" == "x86_64" ]]; then
  23. yum install "libasan-4.8.5-44.el7.${HOSTTYPE}" -y
  24. yum install "libubsan-7.3.1-5.10.el7.${HOSTTYPE}" -y
  25. yum install "devtoolset-8-libasan-devel.${HOSTTYPE}" -y
  26. fi
  27. java -version
  28. JAVA_BIN="$(readlink -f "$(command -v java)")"
  29. echo "java_bin path ${JAVA_BIN}"
  30. export JAVA_HOME="${JAVA_BIN%jre/bin/java}"
  31. /ray/ci/env/install-bazel.sh
  32. # Put bazel into the PATH if building Bazel from source
  33. # export PATH=/root/bazel-3.2.0/output:$PATH:/root/bin
  34. # If converting down to manylinux2010, the following configuration should
  35. # be set for bazel
  36. #echo "build --config=manylinux2010" >> /root/.bazelrc
  37. echo "build --incompatible_linkopts_to_linklibs" >> /root/.bazelrc
  38. if [[ -n "${RAY_INSTALL_JAVA:-}" ]]; then
  39. bazel build //java:ray_java_pkg
  40. unset RAY_INSTALL_JAVA
  41. fi
  42. # Install and use the latest version of Node.js in order to build the dashboard.
  43. set +x
  44. curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
  45. source "$HOME"/.nvm/nvm.sh
  46. nvm install "$NODE_VERSION"
  47. nvm use "$NODE_VERSION"
  48. # Build the dashboard so its static assets can be included in the wheel.
  49. # TODO(mfitton): switch this back when deleting old dashboard code.
  50. (
  51. cd python/ray/dashboard/client
  52. npm ci
  53. npm run build
  54. )
  55. set -x
  56. # Add the repo folder to the safe.dictory global variable to avoid the failure
  57. # because of secruity check from git, when executing the following command
  58. # `git clean ...`, while building wheel locally.
  59. git config --global --add safe.directory /ray
  60. mkdir -p .whl
  61. for PYTHON_NUMPY in "${PYTHON_NUMPYS[@]}" ; do
  62. PYTHON_VERSION_KEY="$(echo "${PYTHON_NUMPY}" | cut -d' ' -f1)"
  63. if [[ "${BUILD_ONE_PYTHON_ONLY:-}" != "" && "${PYTHON_VERSION_KEY}" != "${BUILD_ONE_PYTHON_ONLY}" ]]; then
  64. continue
  65. fi
  66. PYTHON="$(echo "${PYTHON_NUMPY}" | cut -d' ' -f2)"
  67. NUMPY_VERSION="$(echo "${PYTHON_NUMPY}" | cut -d' ' -f3)"
  68. echo "---- Build wheel for ${PYTHON}, numpy=${NUMPY_VERSION}"
  69. # The -f flag is passed twice to also run git clean in the arrow subdirectory.
  70. # The -d flag removes directories. The -x flag ignores the .gitignore file,
  71. # and the -e flag ensures that we don't remove the .whl directory, the
  72. # dashboard directory and jars directory.
  73. git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars
  74. (
  75. cd python
  76. # Fix the numpy version because this will be the oldest numpy version we can
  77. # support.
  78. /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.32
  79. # Set the commit SHA in __init__.py.
  80. if [[ -n "$TRAVIS_COMMIT" ]]; then
  81. sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
  82. else
  83. echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__."
  84. exit 1
  85. fi
  86. # build ray wheel
  87. PATH="/opt/python/${PYTHON}/bin:/root/bazel-3.2.0/output:$PATH" \
  88. "/opt/python/${PYTHON}/bin/python" setup.py bdist_wheel
  89. # build ray-cpp wheel
  90. PATH="/opt/python/${PYTHON}/bin:/root/bazel-3.2.0/output:$PATH" \
  91. RAY_INSTALL_CPP=1 "/opt/python/${PYTHON}/bin/python" setup.py bdist_wheel
  92. # In the future, run auditwheel here.
  93. mv dist/*.whl ../.whl/
  94. )
  95. done
  96. # Rename the wheels so that they can be uploaded to PyPI. TODO(rkn): This is a
  97. # hack, we should use auditwheel instead.
  98. for path in .whl/*.whl; do
  99. if [[ -f "${path}" ]]; then
  100. out="${path//-linux/-manylinux2014}"
  101. if [[ "$out" != "$path" ]]; then
  102. mv "${path}" "${out}"
  103. fi
  104. fi
  105. done
  106. # Clean the build output so later operations is on a clean directory.
  107. git clean -f -f -x -d -e .whl -e python/ray/dashboard/client