core_build.sh 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. #!/usr/bin/env bash
  2. # Licensed to the LF AI & Data foundation under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. The ASF licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. # Compile jobs variable; Usage: $ jobs=12 ./core_build.sh ...
  18. if [[ ! ${jobs+1} ]]; then
  19. if command -v nproc &> /dev/null
  20. # For linux
  21. then
  22. jobs=$(nproc)
  23. elif command -v sysctl &> /dev/null
  24. # For macOS
  25. then
  26. jobs=$(sysctl -n hw.logicalcpu)
  27. else
  28. jobs=4
  29. fi
  30. fi
  31. function get_cpu_arch {
  32. local CPU_ARCH=$1
  33. local OS
  34. OS=$(uname)
  35. local MACHINE
  36. MACHINE=$(uname -m)
  37. ADDITIONAL_FLAGS=""
  38. if [ -z "$CPU_ARCH" ]; then
  39. if [ "$OS" = "Darwin" ]; then
  40. if [ "$MACHINE" = "x86_64" ]; then
  41. local CPU_CAPABILITIES
  42. CPU_CAPABILITIES=$(sysctl -a | grep machdep.cpu.features | awk '{print tolower($0)}')
  43. if [[ $CPU_CAPABILITIES =~ "avx" ]]; then
  44. CPU_ARCH="avx"
  45. else
  46. CPU_ARCH="sse"
  47. fi
  48. elif [[ $(sysctl -a | grep machdep.cpu.brand_string) =~ "Apple" ]]; then
  49. # Apple silicon.
  50. CPU_ARCH="arm64"
  51. fi
  52. else [ "$OS" = "Linux" ];
  53. local CPU_CAPABILITIES
  54. CPU_CAPABILITIES=$(cat /proc/cpuinfo | grep flags | head -n 1| awk '{print tolower($0)}')
  55. if [[ "$CPU_CAPABILITIES" =~ "avx" ]]; then
  56. CPU_ARCH="avx"
  57. elif [[ "$CPU_CAPABILITIES" =~ "sse" ]]; then
  58. CPU_ARCH="sse"
  59. elif [ "$MACHINE" = "aarch64" ]; then
  60. CPU_ARCH="aarch64"
  61. fi
  62. fi
  63. fi
  64. echo -n $CPU_ARCH
  65. }
  66. SOURCE="${BASH_SOURCE[0]}"
  67. while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  68. DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
  69. SOURCE="$(readlink "$SOURCE")"
  70. [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
  71. done
  72. ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
  73. CPP_SRC_DIR="${ROOT_DIR}/internal/core"
  74. BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build"
  75. BUILD_TYPE="Release"
  76. BUILD_UNITTEST="OFF"
  77. INSTALL_PREFIX="${CPP_SRC_DIR}/output"
  78. BUILD_COVERAGE="OFF"
  79. RUN_CPPLINT="OFF"
  80. CUDA_COMPILER=/usr/local/cuda/bin/nvcc
  81. GPU_VERSION="OFF" #defaults to CPU version
  82. CUDA_ARCH="DEFAULT"
  83. EMBEDDED_MILVUS="OFF"
  84. BUILD_DISK_ANN="OFF"
  85. USE_ASAN="OFF"
  86. USE_DYNAMIC_SIMD="ON"
  87. USE_OPENDAL="OFF"
  88. INDEX_ENGINE="KNOWHERE"
  89. : "${ENABLE_GCP_NATIVE:="OFF"}"
  90. while getopts "p:d:t:s:f:n:i:y:a:x:o:ulrcghzmebZ" arg; do
  91. case $arg in
  92. p)
  93. INSTALL_PREFIX=$OPTARG
  94. ;;
  95. t)
  96. BUILD_TYPE=$OPTARG # BUILD_TYPE
  97. ;;
  98. u)
  99. echo "Build and run unittest cases"
  100. BUILD_UNITTEST="ON"
  101. ;;
  102. l)
  103. RUN_CPPLINT="ON"
  104. ;;
  105. c)
  106. BUILD_COVERAGE="ON"
  107. ;;
  108. g)
  109. GPU_VERSION="ON"
  110. ;;
  111. s)
  112. CUDA_ARCH=$OPTARG
  113. ;;
  114. b)
  115. EMBEDDED_MILVUS="ON"
  116. ;;
  117. n)
  118. BUILD_DISK_ANN=$OPTARG
  119. ;;
  120. a)
  121. ENV_VAL=$OPTARG
  122. if [[ ${ENV_VAL} == 'ON' ]]; then
  123. echo "Set USE_ASAN to ON"
  124. USE_ASAN="ON"
  125. BUILD_TYPE=Debug
  126. fi
  127. ;;
  128. y)
  129. USE_DYNAMIC_SIMD=$OPTARG
  130. ;;
  131. Z)
  132. BUILD_WITHOUT_AZURE="on"
  133. ;;
  134. x)
  135. INDEX_ENGINE=$OPTARG
  136. ;;
  137. o)
  138. USE_OPENDAL=$OPTARG
  139. ;;
  140. h) # help
  141. echo "
  142. parameter:
  143. -p: install prefix(default: $(pwd)/milvus)
  144. -d: db data path(default: /tmp/milvus)
  145. -t: build type(default: Debug)
  146. -u: building unit test options(default: OFF)
  147. -l: run cpplint, clang-format and clang-tidy(default: OFF)
  148. -c: code coverage(default: OFF)
  149. -g: build GPU version(default: OFF)
  150. -e: build without prometheus(default: OFF)
  151. -s: build with CUDA arch(default:DEFAULT), for example '-gencode=compute_61,code=sm_61;-gencode=compute_75,code=sm_75'
  152. -b: build embedded milvus(default: OFF)
  153. -a: build milvus with AddressSanitizer(default: false)
  154. -Z: build milvus without azure-sdk-for-cpp, so cannot use azure blob
  155. -o: build milvus with opendal(default: false)
  156. -h: help
  157. usage:
  158. ./core_build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] [-b] [-o]
  159. "
  160. exit 0
  161. ;;
  162. ?)
  163. echo "ERROR! unknown argument"
  164. exit 1
  165. ;;
  166. esac
  167. done
  168. if [ -z "$BUILD_WITHOUT_AZURE" ]; then
  169. AZURE_BUILD_DIR="${ROOT_DIR}/cmake_build/azure"
  170. if [ ! -d ${AZURE_BUILD_DIR} ]; then
  171. mkdir -p ${AZURE_BUILD_DIR}
  172. fi
  173. pushd ${AZURE_BUILD_DIR}
  174. env bash ${ROOT_DIR}/scripts/azure_build.sh -p ${INSTALL_PREFIX} -s ${ROOT_DIR}/internal/core/src/storage/azure-blob-storage -t ${BUILD_UNITTEST}
  175. if [ ! -e libblob-chunk-manager* ]; then
  176. echo "build blob-chunk-manager fail..."
  177. cat vcpkg-bootstrap.log
  178. exit 1
  179. fi
  180. if [ ! -e ${INSTALL_PREFIX}/lib/libblob-chunk-manager* ]; then
  181. echo "install blob-chunk-manager fail..."
  182. exit 1
  183. fi
  184. popd
  185. SYSTEM_NAME=$(uname -s)
  186. if [[ ${SYSTEM_NAME} == "Darwin" ]]; then
  187. SYSTEM_NAME="osx"
  188. elif [[ ${SYSTEM_NAME} == "Linux" ]]; then
  189. SYSTEM_NAME="linux"
  190. fi
  191. ARCHITECTURE=$(uname -m)
  192. if [[ ${ARCHITECTURE} == "x86_64" ]]; then
  193. ARCHITECTURE="x64"
  194. elif [[ ${ARCHITECTURE} == "aarch64" ]]; then
  195. ARCHITECTURE="arm64"
  196. fi
  197. VCPKG_TARGET_TRIPLET=${ARCHITECTURE}-${SYSTEM_NAME}
  198. fi
  199. if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
  200. mkdir ${BUILD_OUTPUT_DIR}
  201. fi
  202. source ${ROOT_DIR}/scripts/setenv.sh
  203. CMAKE_GENERATOR="Unix Makefiles"
  204. # build with diskann index if OS is ubuntu or rocky or amzn
  205. if [ -f /etc/os-release ]; then
  206. . /etc/os-release
  207. OS=$ID
  208. fi
  209. if [ "$OS" = "ubuntu" ] || [ "$OS" = "rocky" ] || [ "$OS" = "amzn" ]; then
  210. BUILD_DISK_ANN=ON
  211. fi
  212. pushd ${BUILD_OUTPUT_DIR}
  213. # Remove make cache since build.sh -l use default variables
  214. # Force update the variables each time
  215. make rebuild_cache >/dev/null 2>&1
  216. CPU_ARCH=$(get_cpu_arch $CPU_TARGET)
  217. arch=$(uname -m)
  218. CMAKE_CMD="cmake \
  219. ${CMAKE_EXTRA_ARGS} \
  220. -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
  221. -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
  222. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
  223. -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
  224. -DCMAKE_LIBRARY_ARCHITECTURE=${arch} \
  225. -DBUILD_COVERAGE=${BUILD_COVERAGE} \
  226. -DMILVUS_GPU_VERSION=${GPU_VERSION} \
  227. -DMILVUS_CUDA_ARCH=${CUDA_ARCH} \
  228. -DEMBEDDED_MILVUS=${EMBEDDED_MILVUS} \
  229. -DBUILD_DISK_ANN=${BUILD_DISK_ANN} \
  230. -DUSE_ASAN=${USE_ASAN} \
  231. -DUSE_DYNAMIC_SIMD=${USE_DYNAMIC_SIMD} \
  232. -DCPU_ARCH=${CPU_ARCH} \
  233. -DUSE_OPENDAL=${USE_OPENDAL} \
  234. -DINDEX_ENGINE=${INDEX_ENGINE} \
  235. -DENABLE_GCP_NATIVE=${ENABLE_GCP_NATIVE} "
  236. if [ -z "$BUILD_WITHOUT_AZURE" ]; then
  237. CMAKE_CMD=${CMAKE_CMD}"-DAZURE_BUILD_DIR=${AZURE_BUILD_DIR} \
  238. -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} "
  239. fi
  240. CMAKE_CMD=${CMAKE_CMD}"${CPP_SRC_DIR}"
  241. echo "CC $CC"
  242. echo ${CMAKE_CMD}
  243. ${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
  244. set
  245. if [[ ${RUN_CPPLINT} == "ON" ]]; then
  246. # cpplint check
  247. make lint
  248. if [ $? -ne 0 ]; then
  249. echo "ERROR! cpplint check failed"
  250. exit 1
  251. fi
  252. echo "cpplint check passed!"
  253. # clang-format check
  254. make check-clang-format
  255. if [ $? -ne 0 ]; then
  256. echo "ERROR! clang-format check failed"
  257. exit 1
  258. fi
  259. echo "clang-format check passed!"
  260. else
  261. # compile and build
  262. make -j ${jobs} install || exit 1
  263. fi
  264. if command -v ccache &> /dev/null
  265. then
  266. ccache -s
  267. fi
  268. popd