util_insert.go 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. // Licensed to the LF AI & Data foundation under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. The ASF licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. package integration
  17. import (
  18. "context"
  19. "time"
  20. "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
  21. "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
  22. "github.com/milvus-io/milvus/pkg/util/testutils"
  23. )
  24. func (s *MiniClusterSuite) WaitForFlush(ctx context.Context, segIDs []int64, flushTs uint64, dbName, collectionName string) {
  25. flushed := func() bool {
  26. resp, err := s.Cluster.Proxy.GetFlushState(ctx, &milvuspb.GetFlushStateRequest{
  27. SegmentIDs: segIDs,
  28. FlushTs: flushTs,
  29. DbName: dbName,
  30. CollectionName: collectionName,
  31. })
  32. if err != nil {
  33. return false
  34. }
  35. return resp.GetFlushed()
  36. }
  37. for !flushed() {
  38. select {
  39. case <-ctx.Done():
  40. s.FailNow("failed to wait for flush until ctx done")
  41. return
  42. default:
  43. time.Sleep(500 * time.Millisecond)
  44. }
  45. }
  46. }
  47. func NewInt64FieldData(fieldName string, numRows int) *schemapb.FieldData {
  48. return &schemapb.FieldData{
  49. Type: schemapb.DataType_Int64,
  50. FieldName: fieldName,
  51. Field: &schemapb.FieldData_Scalars{
  52. Scalars: &schemapb.ScalarField{
  53. Data: &schemapb.ScalarField_LongData{
  54. LongData: &schemapb.LongArray{
  55. Data: GenerateInt64Array(numRows, 0),
  56. },
  57. },
  58. },
  59. },
  60. }
  61. }
  62. func NewInt64FieldDataWithStart(fieldName string, numRows int, start int64) *schemapb.FieldData {
  63. return &schemapb.FieldData{
  64. Type: schemapb.DataType_Int64,
  65. FieldName: fieldName,
  66. Field: &schemapb.FieldData_Scalars{
  67. Scalars: &schemapb.ScalarField{
  68. Data: &schemapb.ScalarField_LongData{
  69. LongData: &schemapb.LongArray{
  70. Data: GenerateInt64Array(numRows, start),
  71. },
  72. },
  73. },
  74. },
  75. }
  76. }
  77. func NewInt64FieldDataNullableWithStart(fieldName string, numRows, start int) *schemapb.FieldData {
  78. validData, num := GenerateBoolArray(numRows)
  79. return &schemapb.FieldData{
  80. Type: schemapb.DataType_Int64,
  81. FieldName: fieldName,
  82. Field: &schemapb.FieldData_Scalars{
  83. Scalars: &schemapb.ScalarField{
  84. Data: &schemapb.ScalarField_LongData{
  85. LongData: &schemapb.LongArray{
  86. Data: GenerateInt64Array(num, int64(start)),
  87. },
  88. },
  89. },
  90. },
  91. ValidData: validData,
  92. }
  93. }
  94. func NewInt64SameFieldData(fieldName string, numRows int, value int64) *schemapb.FieldData {
  95. return &schemapb.FieldData{
  96. Type: schemapb.DataType_Int64,
  97. FieldName: fieldName,
  98. Field: &schemapb.FieldData_Scalars{
  99. Scalars: &schemapb.ScalarField{
  100. Data: &schemapb.ScalarField_LongData{
  101. LongData: &schemapb.LongArray{
  102. Data: GenerateSameInt64Array(numRows, value),
  103. },
  104. },
  105. },
  106. },
  107. }
  108. }
  109. func NewVarCharSameFieldData(fieldName string, numRows int, value string) *schemapb.FieldData {
  110. return &schemapb.FieldData{
  111. Type: schemapb.DataType_String,
  112. FieldName: fieldName,
  113. Field: &schemapb.FieldData_Scalars{
  114. Scalars: &schemapb.ScalarField{
  115. Data: &schemapb.ScalarField_StringData{
  116. StringData: &schemapb.StringArray{
  117. Data: GenerateSameStringArray(numRows, value),
  118. },
  119. },
  120. },
  121. },
  122. }
  123. }
  124. func NewStringFieldData(fieldName string, numRows int) *schemapb.FieldData {
  125. return testutils.NewStringFieldData(fieldName, numRows)
  126. }
  127. func NewFloatVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
  128. return testutils.NewFloatVectorFieldData(fieldName, numRows, dim)
  129. }
  130. func NewFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
  131. return testutils.NewFloat16VectorFieldData(fieldName, numRows, dim)
  132. }
  133. func NewBFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
  134. return testutils.NewBFloat16VectorFieldData(fieldName, numRows, dim)
  135. }
  136. func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
  137. return testutils.NewBinaryVectorFieldData(fieldName, numRows, dim)
  138. }
  139. func NewSparseFloatVectorFieldData(fieldName string, numRows int) *schemapb.FieldData {
  140. return testutils.NewSparseFloatVectorFieldData(fieldName, numRows)
  141. }
  142. func GenerateInt64Array(numRows int, start int64) []int64 {
  143. ret := make([]int64, numRows)
  144. for i := 0; i < numRows; i++ {
  145. ret[i] = int64(i) + start
  146. }
  147. return ret
  148. }
  149. func GenerateSameInt64Array(numRows int, value int64) []int64 {
  150. ret := make([]int64, numRows)
  151. for i := 0; i < numRows; i++ {
  152. ret[i] = value
  153. }
  154. return ret
  155. }
  156. func GenerateBoolArray(numRows int) ([]bool, int) {
  157. var num int
  158. ret := make([]bool, numRows)
  159. for i := 0; i < numRows; i++ {
  160. ret[i] = i%2 == 0
  161. if ret[i] {
  162. num++
  163. }
  164. }
  165. return ret, num
  166. }
  167. func GenerateSameStringArray(numRows int, value string) []string {
  168. ret := make([]string, numRows)
  169. for i := 0; i < numRows; i++ {
  170. ret[i] = value
  171. }
  172. return ret
  173. }
  174. func GenerateSparseFloatArray(numRows int) *schemapb.SparseFloatArray {
  175. return testutils.GenerateSparseFloatVectors(numRows)
  176. }
  177. func GenerateHashKeys(numRows int) []uint32 {
  178. return testutils.GenerateHashKeys(numRows)
  179. }