util_index.go 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. // Licensed to the LF AI & Data foundation under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. The ASF licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. package integration
  17. import (
  18. "context"
  19. "fmt"
  20. "strconv"
  21. "testing"
  22. "time"
  23. "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
  24. "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
  25. "github.com/milvus-io/milvus/pkg/common"
  26. "github.com/milvus-io/milvus/pkg/util/indexparamcheck"
  27. )
  28. const (
  29. IndexRaftIvfFlat = indexparamcheck.IndexRaftIvfFlat
  30. IndexRaftIvfPQ = indexparamcheck.IndexRaftIvfPQ
  31. IndexFaissIDMap = indexparamcheck.IndexFaissIDMap
  32. IndexFaissIvfFlat = indexparamcheck.IndexFaissIvfFlat
  33. IndexFaissIvfPQ = indexparamcheck.IndexFaissIvfPQ
  34. IndexScaNN = indexparamcheck.IndexScaNN
  35. IndexFaissIvfSQ8 = indexparamcheck.IndexFaissIvfSQ8
  36. IndexFaissBinIDMap = indexparamcheck.IndexFaissBinIDMap
  37. IndexFaissBinIvfFlat = indexparamcheck.IndexFaissBinIvfFlat
  38. IndexHNSW = indexparamcheck.IndexHNSW
  39. IndexDISKANN = indexparamcheck.IndexDISKANN
  40. IndexSparseInvertedIndex = indexparamcheck.IndexSparseInverted
  41. IndexSparseWand = indexparamcheck.IndexSparseWand
  42. )
  43. func (s *MiniClusterSuite) WaitForIndexBuiltWithDB(ctx context.Context, dbName, collection, field string) {
  44. s.waitForIndexBuiltInternal(ctx, dbName, collection, field, "")
  45. }
  46. func (s *MiniClusterSuite) WaitForIndexBuilt(ctx context.Context, collection, field string) {
  47. s.waitForIndexBuiltInternal(ctx, "", collection, field, "")
  48. }
  49. func (s *MiniClusterSuite) WaitForIndexBuiltWithIndexName(ctx context.Context, collection, field, indexName string) {
  50. s.waitForIndexBuiltInternal(ctx, "", collection, field, indexName)
  51. }
  52. func (s *MiniClusterSuite) waitForIndexBuiltInternal(ctx context.Context, dbName, collection, field, indexName string) {
  53. getIndexBuilt := func() bool {
  54. resp, err := s.Cluster.Proxy.DescribeIndex(ctx, &milvuspb.DescribeIndexRequest{
  55. DbName: dbName,
  56. CollectionName: collection,
  57. FieldName: field,
  58. IndexName: indexName,
  59. })
  60. if err != nil {
  61. s.FailNow("failed to describe index")
  62. return true
  63. }
  64. for _, desc := range resp.GetIndexDescriptions() {
  65. if desc.GetFieldName() == field {
  66. switch desc.GetState() {
  67. case commonpb.IndexState_Finished:
  68. return true
  69. case commonpb.IndexState_Failed:
  70. return false
  71. }
  72. }
  73. }
  74. return false
  75. }
  76. for !getIndexBuilt() {
  77. select {
  78. case <-ctx.Done():
  79. s.FailNow("failed to wait index built until ctx done")
  80. return
  81. case <-time.After(500 * time.Millisecond):
  82. }
  83. }
  84. }
  85. func waitingForIndexBuilt(ctx context.Context, cluster *MiniClusterV2, t *testing.T, collection, field string) {
  86. getIndexBuilt := func() bool {
  87. resp, err := cluster.Proxy.DescribeIndex(ctx, &milvuspb.DescribeIndexRequest{
  88. CollectionName: collection,
  89. FieldName: field,
  90. })
  91. if err != nil {
  92. t.FailNow()
  93. return true
  94. }
  95. for _, desc := range resp.GetIndexDescriptions() {
  96. if desc.GetFieldName() == field {
  97. switch desc.GetState() {
  98. case commonpb.IndexState_Finished:
  99. return true
  100. case commonpb.IndexState_Failed:
  101. return false
  102. }
  103. }
  104. }
  105. return false
  106. }
  107. for !getIndexBuilt() {
  108. select {
  109. case <-ctx.Done():
  110. t.FailNow()
  111. return
  112. case <-time.After(500 * time.Millisecond):
  113. }
  114. }
  115. }
  116. func ConstructIndexParam(dim int, indexType string, metricType string) []*commonpb.KeyValuePair {
  117. params := []*commonpb.KeyValuePair{
  118. {
  119. Key: common.DimKey,
  120. Value: strconv.Itoa(dim),
  121. },
  122. {
  123. Key: common.MetricTypeKey,
  124. Value: metricType,
  125. },
  126. {
  127. Key: common.IndexTypeKey,
  128. Value: indexType,
  129. },
  130. }
  131. switch indexType {
  132. case IndexFaissIDMap, IndexFaissBinIDMap:
  133. // no index param is required
  134. case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexScaNN:
  135. params = append(params, &commonpb.KeyValuePair{
  136. Key: "nlist",
  137. Value: "100",
  138. })
  139. case IndexFaissIvfPQ:
  140. params = append(params, &commonpb.KeyValuePair{
  141. Key: "nlist",
  142. Value: "100",
  143. })
  144. params = append(params, &commonpb.KeyValuePair{
  145. Key: "m",
  146. Value: "16",
  147. })
  148. params = append(params, &commonpb.KeyValuePair{
  149. Key: "nbits",
  150. Value: "8",
  151. })
  152. case IndexHNSW:
  153. params = append(params, &commonpb.KeyValuePair{
  154. Key: "M",
  155. Value: "16",
  156. })
  157. params = append(params, &commonpb.KeyValuePair{
  158. Key: "efConstruction",
  159. Value: "200",
  160. })
  161. case IndexSparseInvertedIndex:
  162. case IndexSparseWand:
  163. case IndexDISKANN:
  164. default:
  165. panic(fmt.Sprintf("unimplemented index param for %s, please help to improve it", indexType))
  166. }
  167. return params
  168. }
  169. func GetSearchParams(indexType string, metricType string) map[string]any {
  170. params := make(map[string]any)
  171. switch indexType {
  172. case IndexFaissIDMap, IndexFaissBinIDMap:
  173. params[common.MetricTypeKey] = metricType
  174. case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexFaissIvfPQ, IndexScaNN:
  175. params["nprobe"] = 8
  176. case IndexHNSW:
  177. params["ef"] = 200
  178. case IndexDISKANN:
  179. params["search_list"] = 20
  180. case IndexSparseInvertedIndex:
  181. case IndexSparseWand:
  182. params["drop_ratio_search"] = 0.1
  183. default:
  184. panic(fmt.Sprintf("unimplemented search param for %s, please help to improve it", indexType))
  185. }
  186. return params
  187. }