utils.go 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. package common
  2. import (
  3. "encoding/binary"
  4. "fmt"
  5. "math"
  6. "math/rand"
  7. "strings"
  8. "time"
  9. "github.com/x448/float16"
  10. "go.uber.org/zap"
  11. "github.com/milvus-io/milvus/client/v2/entity"
  12. "github.com/milvus-io/milvus/pkg/log"
  13. )
  14. var (
  15. letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
  16. r *rand.Rand
  17. )
  18. func init() {
  19. r = rand.New(rand.NewSource(time.Now().UnixNano()))
  20. }
  21. func GenRandomString(prefix string, n int) string {
  22. b := make([]rune, n)
  23. for i := range b {
  24. b[i] = letterRunes[r.Intn(len(letterRunes))]
  25. }
  26. str := fmt.Sprintf("%s_%s", prefix, string(b))
  27. return str
  28. }
  29. // GenLongString gen invalid long string
  30. func GenLongString(n int) string {
  31. var builder strings.Builder
  32. longString := "a"
  33. for i := 0; i < n; i++ {
  34. builder.WriteString(longString)
  35. }
  36. return builder.String()
  37. }
  38. func GenValidNames() []string {
  39. return []string{
  40. "a",
  41. "_",
  42. "_name",
  43. "_123",
  44. "name_",
  45. "_coll_123_",
  46. }
  47. }
  48. func GenInvalidNames() []string {
  49. invalidNames := []string{
  50. "",
  51. " ",
  52. "12-s",
  53. "(mn)",
  54. "中文",
  55. "%$#",
  56. "1",
  57. "[10]",
  58. "a b",
  59. DefaultDynamicFieldName,
  60. GenLongString(MaxCollectionNameLen + 1),
  61. }
  62. return invalidNames
  63. }
  64. func GenFloatVector(dim int) []float32 {
  65. vector := make([]float32, 0, dim)
  66. for j := 0; j < dim; j++ {
  67. vector = append(vector, rand.Float32())
  68. }
  69. return vector
  70. }
  71. func GenFloat16Vector(dim int) []byte {
  72. ret := make([]byte, dim*2)
  73. for i := 0; i < dim; i++ {
  74. v := float16.Fromfloat32(rand.Float32()).Bits()
  75. binary.LittleEndian.PutUint16(ret[i*2:], v)
  76. }
  77. return ret
  78. }
  79. func GenBFloat16Vector(dim int) []byte {
  80. ret16 := make([]uint16, 0, dim)
  81. for i := 0; i < dim; i++ {
  82. f := rand.Float32()
  83. bits := math.Float32bits(f)
  84. bits >>= 16
  85. bits &= 0x7FFF
  86. ret16 = append(ret16, uint16(bits))
  87. }
  88. ret := make([]byte, len(ret16)*2)
  89. for i, value := range ret16 {
  90. binary.LittleEndian.PutUint16(ret[i*2:], value)
  91. }
  92. return ret
  93. }
  94. func GenBinaryVector(dim int) []byte {
  95. vector := make([]byte, dim/8)
  96. rand.Read(vector)
  97. return vector
  98. }
  99. func GenSparseVector(maxLen int) entity.SparseEmbedding {
  100. length := 1 + rand.Intn(1+maxLen)
  101. positions := make([]uint32, length)
  102. values := make([]float32, length)
  103. for i := 0; i < length; i++ {
  104. positions[i] = uint32(2*i + 1)
  105. values[i] = rand.Float32()
  106. }
  107. vector, err := entity.NewSliceSparseEmbedding(positions, values)
  108. if err != nil {
  109. log.Fatal("Generate vector failed %s", zap.Error(err))
  110. }
  111. return vector
  112. }
  113. // InvalidExprStruct invalid expr
  114. type InvalidExprStruct struct {
  115. Expr string
  116. ErrNil bool
  117. ErrMsg string
  118. }
  119. var InvalidExpressions = []InvalidExprStruct{
  120. {Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error
  121. {Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword
  122. {Expr: "int64 > 10 AND int64 < 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // AND isn't supported
  123. {Expr: "int64 < 10 OR int64 > 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // OR isn't supported
  124. {Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field
  125. {Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch
  126. {Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty
  127. {Expr: fmt.Sprintf("%s like 'a%%' ", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty
  128. {Expr: fmt.Sprintf("%s like `a%%` ", DefaultJSONFieldName), ErrNil: false, ErrMsg: "cannot parse expression"}, // ``
  129. {Expr: fmt.Sprintf("%s > 1", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""}, // hits empty
  130. {Expr: fmt.Sprintf("%s[\"dynamicList\"] == [2, 3]", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""},
  131. {Expr: fmt.Sprintf("%s['a'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // json field not exist
  132. {Expr: fmt.Sprintf("%s['number'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match
  133. {Expr: fmt.Sprintf("%s[0] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match
  134. {Expr: fmt.Sprintf("json_contains (%s['number'], 2)", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
  135. {Expr: fmt.Sprintf("json_contains (%s['list'], [2])", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
  136. {Expr: fmt.Sprintf("json_contains_all (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_all operation element must be an array"},
  137. {Expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_any operation element must be an array"},
  138. {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"},
  139. {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"},
  140. {Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] >
  141. {Expr: fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] >
  142. }