conftest.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489
  1. import logging
  2. import pytest
  3. import functools
  4. import socket
  5. import common.common_type as ct
  6. import common.common_func as cf
  7. from utils.util_log import test_log as log
  8. from common.common_func import param_info
  9. from check.param_check import ip_check, number_check
  10. from config.log_config import log_config
  11. from utils.util_pymilvus import get_milvus, gen_unique_str, gen_default_fields, gen_binary_default_fields
  12. from pymilvus.orm.types import CONSISTENCY_STRONG
  13. timeout = 60
  14. dimension = 128
  15. delete_timeout = 60
  16. def pytest_addoption(parser):
  17. parser.addoption("--host", action="store", default="localhost", help="service's ip")
  18. parser.addoption("--service", action="store", default="", help="service address")
  19. parser.addoption("--port", action="store", default=19530, help="service's port")
  20. parser.addoption("--user", action="store", default="", help="user name for connection")
  21. parser.addoption("--password", action="store", default="", help="password for connection")
  22. parser.addoption("--db_name", action="store", default="default", help="database name for connection")
  23. parser.addoption("--secure", type=bool, action="store", default=False, help="secure for connection")
  24. parser.addoption("--milvus_ns", action="store", default="chaos-testing", help="milvus_ns")
  25. parser.addoption("--http_port", action="store", default=19121, help="http's port")
  26. parser.addoption("--handler", action="store", default="GRPC", help="handler of request")
  27. parser.addoption("--tag", action="store", default="all", help="only run tests matching the tag.")
  28. parser.addoption('--dry_run', action='store_true', default=False, help="")
  29. parser.addoption('--database_name', action='store', default="default", help="name of database")
  30. parser.addoption('--partition_name', action='store', default="partition_name", help="name of partition")
  31. parser.addoption('--connect_name', action='store', default="connect_name", help="name of connect")
  32. parser.addoption('--descriptions', action='store', default="partition_des", help="descriptions of partition")
  33. parser.addoption('--collection_name', action='store', default="collection_name", help="name of collection")
  34. parser.addoption('--search_vectors', action='store', default="search_vectors", help="vectors of search")
  35. parser.addoption('--index_param', action='store', default="index_param", help="index_param of index")
  36. parser.addoption('--data', action='store', default="data", help="data of request")
  37. parser.addoption('--clean_log', action='store_true', default=False, help="clean log before testing")
  38. parser.addoption('--schema', action='store', default="schema", help="schema of test interface")
  39. parser.addoption('--err_msg', action='store', default="err_msg", help="error message of test")
  40. parser.addoption('--term_expr', action='store', default="term_expr", help="expr of query quest")
  41. parser.addoption('--check_content', action='store', default="check_content", help="content of check")
  42. parser.addoption('--field_name', action='store', default="field_name", help="field_name of index")
  43. parser.addoption('--replica_num', type='int', action='store', default=ct.default_replica_num, help="memory replica number")
  44. parser.addoption('--minio_host', action='store', default="localhost", help="minio service's ip")
  45. parser.addoption('--uri', action='store', default="", help="uri for high level api")
  46. parser.addoption('--token', action='store', default="", help="token for high level api")
  47. parser.addoption("--request_duration", action="store", default="10m", help="request_duration")
  48. @pytest.fixture
  49. def host(request):
  50. return request.config.getoption("--host")
  51. @pytest.fixture
  52. def service(request):
  53. return request.config.getoption("--service")
  54. @pytest.fixture
  55. def port(request):
  56. return request.config.getoption("--port")
  57. @pytest.fixture
  58. def user(request):
  59. return request.config.getoption("--user")
  60. @pytest.fixture
  61. def password(request):
  62. return request.config.getoption("--password")
  63. @pytest.fixture
  64. def db_name(request):
  65. return request.config.getoption("--db_name")
  66. @pytest.fixture
  67. def secure(request):
  68. return request.config.getoption("--secure")
  69. @pytest.fixture
  70. def milvus_ns(request):
  71. return request.config.getoption("--milvus_ns")
  72. @pytest.fixture
  73. def http_port(request):
  74. return request.config.getoption("--http_port")
  75. @pytest.fixture
  76. def handler(request):
  77. return request.config.getoption("--handler")
  78. @pytest.fixture
  79. def tag(request):
  80. return request.config.getoption("--tag")
  81. @pytest.fixture
  82. def dry_run(request):
  83. return request.config.getoption("--dry_run")
  84. @pytest.fixture
  85. def connect_name(request):
  86. return request.config.getoption("--connect_name")
  87. @pytest.fixture
  88. def database_name(request):
  89. return request.config.getoption("--database_name")
  90. @pytest.fixture
  91. def partition_name(request):
  92. return request.config.getoption("--partition_name")
  93. @pytest.fixture
  94. def descriptions(request):
  95. return request.config.getoption("--descriptions")
  96. @pytest.fixture
  97. def collection_name(request):
  98. return request.config.getoption("--collection_name")
  99. @pytest.fixture
  100. def search_vectors(request):
  101. return request.config.getoption("--search_vectors")
  102. @pytest.fixture
  103. def index_param(request):
  104. return request.config.getoption("--index_param")
  105. @pytest.fixture
  106. def data(request):
  107. return request.config.getoption("--data")
  108. @pytest.fixture
  109. def clean_log(request):
  110. return request.config.getoption("--clean_log")
  111. @pytest.fixture
  112. def schema(request):
  113. return request.config.getoption("--schema")
  114. @pytest.fixture
  115. def err_msg(request):
  116. return request.config.getoption("--err_msg")
  117. @pytest.fixture
  118. def term_expr(request):
  119. return request.config.getoption("--term_expr")
  120. @pytest.fixture
  121. def check_content(request):
  122. log.error("^" * 50)
  123. log.error("check_content")
  124. return request.config.getoption("--check_content")
  125. @pytest.fixture
  126. def field_name(request):
  127. return request.config.getoption("--field_name")
  128. @pytest.fixture
  129. def minio_host(request):
  130. return request.config.getoption("--minio_host")
  131. @pytest.fixture
  132. def uri(request):
  133. return request.config.getoption("--uri")
  134. @pytest.fixture
  135. def token(request):
  136. return request.config.getoption("--token")
  137. @pytest.fixture
  138. def request_duration(request):
  139. return request.config.getoption("--request_duration")
  140. """ fixture func """
  141. @pytest.fixture(scope="session", autouse=True)
  142. def initialize_env(request):
  143. """ clean log before testing """
  144. host = request.config.getoption("--host")
  145. port = request.config.getoption("--port")
  146. handler = request.config.getoption("--handler")
  147. user = request.config.getoption("--user")
  148. password = request.config.getoption("--password")
  149. secure = request.config.getoption("--secure")
  150. clean_log = request.config.getoption("--clean_log")
  151. replica_num = request.config.getoption("--replica_num")
  152. uri = request.config.getoption("--uri")
  153. token = request.config.getoption("--token")
  154. """ params check """
  155. assert ip_check(host) and number_check(port)
  156. """ modify log files """
  157. file_path_list = [log_config.log_debug, log_config.log_info, log_config.log_err]
  158. if log_config.log_worker != "":
  159. file_path_list.append(log_config.log_worker)
  160. cf.modify_file(file_path_list=file_path_list, is_modify=clean_log)
  161. log.info("#" * 80)
  162. log.info("[initialize_milvus] Log cleaned up, start testing...")
  163. param_info.prepare_param_info(host, port, handler, replica_num, user, password, secure, uri, token)
  164. @pytest.fixture(params=cf.gen_simple_index())
  165. def get_index_param(request):
  166. yield request.param
  167. # TODO: construct invalid index params for all index types
  168. @pytest.fixture(params=[{"metric_type": "L3", "index_type": "IVF_FLAT"},
  169. {"metric_type": "L2", "index_type": "IVF_FLAT", "err_params": {"nlist": 10}},
  170. {"metric_type": "L2", "index_type": "IVF_FLAT", "params": {"nlist": -1}}])
  171. def get_invalid_index_params(request):
  172. yield request.param
  173. @pytest.fixture(params=ct.get_invalid_dict)
  174. def get_invalid_vector_dict(request):
  175. yield request.param
  176. def pytest_configure(config):
  177. # register an additional marker
  178. config.addinivalue_line(
  179. "markers", "tag(name): mark test to run only matching the tag"
  180. )
  181. def pytest_runtest_setup(item):
  182. tags = list()
  183. for marker in item.iter_markers(name="tag"):
  184. for tag in marker.args:
  185. tags.append(tag)
  186. if tags:
  187. cmd_tag = item.config.getoption("--tag")
  188. if cmd_tag != "all" and cmd_tag not in tags:
  189. pytest.skip("test requires tag in {!r}".format(tags))
  190. def pytest_runtestloop(session):
  191. if session.config.getoption('--dry_run'):
  192. total_num = 0
  193. file_num = 0
  194. tags_num = 0
  195. res = {"total_num": total_num, "tags_num": tags_num}
  196. for item in session.items:
  197. print(item.nodeid)
  198. if item.fspath.basename not in res:
  199. res.update({item.fspath.basename: {"total": 1, "tags": 0}})
  200. else:
  201. res[item.fspath.basename]["total"] += 1
  202. res["total_num"] += 1
  203. for marker in item.own_markers:
  204. if marker.name == "tags" and "0331" in marker.args:
  205. res["tags_num"] += 1
  206. res[item.fspath.basename]["tags"] += 1
  207. print(res)
  208. return True
  209. def check_server_connection(request):
  210. host = request.config.getoption("--host")
  211. port = request.config.getoption("--port")
  212. connected = True
  213. if host and (host not in ['localhost', '127.0.0.1']):
  214. try:
  215. socket.getaddrinfo(host, port, 0, 0, socket.IPPROTO_TCP)
  216. except Exception as e:
  217. print("Socket connnet failed: %s" % str(e))
  218. connected = False
  219. return connected
  220. # @pytest.fixture(scope="session", autouse=True)
  221. # def change_mutation_result_to_primary_keys():
  222. # def insert_future_decorator(func):
  223. # @functools.wraps(func)
  224. # def change(*args, **kwargs):
  225. # try:
  226. # return func(*args, **kwargs).primary_keys
  227. # except Exception as e:
  228. # raise e
  229. # return change
  230. #
  231. # from pymilvus import MutationFuture
  232. # MutationFuture.result = insert_future_decorator(MutationFuture.result)
  233. #
  234. # def insert_decorator(func):
  235. # @functools.wraps(func)
  236. # def change(*args, **kwargs):
  237. # if kwargs.get("_async", False):
  238. # return func(*args, **kwargs)
  239. # try:
  240. # return func(*args, **kwargs).primary_keys
  241. # except Exception as e:
  242. # raise e
  243. # return change
  244. # Milvus.insert = insert_decorator(Milvus.insert)
  245. # yield
  246. @pytest.fixture(scope="module")
  247. def connect(request):
  248. host = request.config.getoption("--host")
  249. service_name = request.config.getoption("--service")
  250. port = request.config.getoption("--port")
  251. http_port = request.config.getoption("--http_port")
  252. handler = request.config.getoption("--handler")
  253. if handler == "HTTP":
  254. port = http_port
  255. try:
  256. milvus = get_milvus(host=host, port=port, handler=handler)
  257. # reset_build_index_threshold(milvus)
  258. except Exception as e:
  259. logging.getLogger().error(str(e))
  260. pytest.exit("Milvus server can not connected, exit pytest ...")
  261. def fin():
  262. try:
  263. milvus.close()
  264. pass
  265. except Exception as e:
  266. logging.getLogger().info(str(e))
  267. request.addfinalizer(fin)
  268. return milvus
  269. @pytest.fixture(scope="module")
  270. def dis_connect(request):
  271. host = request.config.getoption("--host")
  272. service_name = request.config.getoption("--service")
  273. port = request.config.getoption("--port")
  274. http_port = request.config.getoption("--http_port")
  275. handler = request.config.getoption("--handler")
  276. if handler == "HTTP":
  277. port = http_port
  278. milvus = get_milvus(host=host, port=port, handler=handler)
  279. milvus.close()
  280. return milvus
  281. @pytest.fixture(scope="module")
  282. def args(request):
  283. host = request.config.getoption("--host")
  284. service_name = request.config.getoption("--service")
  285. port = request.config.getoption("--port")
  286. http_port = request.config.getoption("--http_port")
  287. handler = request.config.getoption("--handler")
  288. if handler == "HTTP":
  289. port = http_port
  290. args = {"ip": host, "port": port, "handler": handler, "service_name": service_name}
  291. return args
  292. @pytest.fixture(scope="module")
  293. def milvus(request):
  294. host = request.config.getoption("--host")
  295. port = request.config.getoption("--port")
  296. http_port = request.config.getoption("--http_port")
  297. handler = request.config.getoption("--handler")
  298. if handler == "HTTP":
  299. port = http_port
  300. return get_milvus(host=host, port=port, handler=handler)
  301. @pytest.fixture(scope="function")
  302. def collection(request, connect):
  303. ori_collection_name = getattr(request.module, "collection_id", "test")
  304. collection_name = gen_unique_str(ori_collection_name)
  305. log.debug(f'collection_name: {collection_name}')
  306. try:
  307. default_fields = gen_default_fields()
  308. connect.create_collection(collection_name, default_fields, consistency_level=CONSISTENCY_STRONG)
  309. except Exception as e:
  310. pytest.exit(str(e))
  311. def teardown():
  312. if connect.has_collection(collection_name):
  313. connect.drop_collection(collection_name, timeout=delete_timeout)
  314. request.addfinalizer(teardown)
  315. assert connect.has_collection(collection_name)
  316. return collection_name
  317. # customised id
  318. @pytest.fixture(scope="function")
  319. def id_collection(request, connect):
  320. ori_collection_name = getattr(request.module, "collection_id", "test")
  321. collection_name = gen_unique_str(ori_collection_name)
  322. log.debug(f'id_collection_name: {collection_name}')
  323. try:
  324. fields = gen_default_fields(auto_id=False)
  325. connect.create_collection(collection_name, fields, consistency_level=CONSISTENCY_STRONG)
  326. except Exception as e:
  327. pytest.exit(str(e))
  328. def teardown():
  329. if connect.has_collection(collection_name):
  330. connect.drop_collection(collection_name, timeout=delete_timeout)
  331. request.addfinalizer(teardown)
  332. assert connect.has_collection(collection_name)
  333. return collection_name
  334. @pytest.fixture(scope="function")
  335. def binary_collection(request, connect):
  336. ori_collection_name = getattr(request.module, "collection_id", "test")
  337. collection_name = gen_unique_str(ori_collection_name)
  338. try:
  339. fields = gen_binary_default_fields()
  340. connect.create_collection(collection_name, fields, consistency_level=CONSISTENCY_STRONG)
  341. except Exception as e:
  342. pytest.exit(str(e))
  343. def teardown():
  344. collection_names = connect.list_collections()
  345. if connect.has_collection(collection_name):
  346. connect.drop_collection(collection_name, timeout=delete_timeout)
  347. request.addfinalizer(teardown)
  348. assert connect.has_collection(collection_name)
  349. return collection_name
  350. # customised id
  351. @pytest.fixture(scope="function")
  352. def binary_id_collection(request, connect):
  353. ori_collection_name = getattr(request.module, "collection_id", "test")
  354. collection_name = gen_unique_str(ori_collection_name)
  355. try:
  356. fields = gen_binary_default_fields(auto_id=False)
  357. connect.create_collection(collection_name, fields, consistency_level=CONSISTENCY_STRONG)
  358. except Exception as e:
  359. pytest.exit(str(e))
  360. def teardown():
  361. if connect.has_collection(collection_name):
  362. connect.drop_collection(collection_name, timeout=delete_timeout)
  363. request.addfinalizer(teardown)
  364. assert connect.has_collection(collection_name)
  365. return collection_name
  366. # for test exit in the future
  367. # @pytest.hookimpl(hookwrapper=True, tryfirst=True)
  368. # def pytest_runtest_makereport():
  369. # result = yield
  370. # report = result.get_result()
  371. # if report.outcome == "failed":
  372. # msg = "The execution of the test case fails and the test exits..."
  373. # log.error(msg)
  374. # pytest.exit(msg)