update_ci_routes.py 2.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. #!/usr/bin/env python3
  2. from functools import lru_cache
  3. import sys
  4. import subprocess
  5. from tqdm import tqdm
  6. from azure.storage.blob import BlockBlobService # pylint: disable=import-error
  7. from selfdrive.car.tests.routes import routes as test_car_models_routes
  8. from selfdrive.test.process_replay.test_processes import source_segments as replay_segments
  9. from xx.chffr.lib import azureutil # pylint: disable=import-error
  10. from xx.chffr.lib.storage import _DATA_ACCOUNT_PRODUCTION, _DATA_ACCOUNT_CI, _DATA_BUCKET_PRODUCTION # pylint: disable=import-error
  11. SOURCES = [
  12. (_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION),
  13. (_DATA_ACCOUNT_CI, "commadataci"),
  14. ]
  15. @lru_cache
  16. def get_azure_keys():
  17. dest_key = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
  18. source_keys = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
  19. service = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=dest_key)
  20. return dest_key, source_keys, service
  21. def upload_route(path, exclude_patterns=None):
  22. dest_key, _, _ = get_azure_keys()
  23. if exclude_patterns is None:
  24. exclude_patterns = ['*/dcamera.hevc']
  25. r, n = path.rsplit("--", 1)
  26. r = '/'.join(r.split('/')[-2:]) # strip out anything extra in the path
  27. destpath = f"{r}/{n}"
  28. cmd = [
  29. "azcopy",
  30. "copy",
  31. f"{path}/*",
  32. f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{dest_key}",
  33. "--recursive=false",
  34. "--overwrite=false",
  35. ] + [f"--exclude-pattern={p}" for p in exclude_patterns]
  36. subprocess.check_call(cmd)
  37. def sync_to_ci_public(route):
  38. dest_key, source_keys, service = get_azure_keys()
  39. key_prefix = route.replace('|', '/')
  40. dongle_id = key_prefix.split('/')[0]
  41. if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None:
  42. return True
  43. print(f"Uploading {route}")
  44. for (source_account, source_bucket), source_key in zip(SOURCES, source_keys):
  45. print(f"Trying {source_account}/{source_bucket}")
  46. cmd = [
  47. "azcopy",
  48. "copy",
  49. f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}",
  50. f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{dest_key}",
  51. "--recursive=true",
  52. "--overwrite=false",
  53. "--exclude-pattern=*/dcamera.hevc",
  54. ]
  55. try:
  56. result = subprocess.call(cmd, stdout=subprocess.DEVNULL)
  57. if result == 0:
  58. print("Success")
  59. return True
  60. except subprocess.CalledProcessError:
  61. print("Failed")
  62. return False
  63. if __name__ == "__main__":
  64. failed_routes = []
  65. to_sync = sys.argv[1:]
  66. if not len(to_sync):
  67. # sync routes from the car tests routes and process replay
  68. to_sync.extend([rt.route for rt in test_car_models_routes])
  69. to_sync.extend([s[1].rsplit('--', 1)[0] for s in replay_segments])
  70. for r in tqdm(to_sync):
  71. if not sync_to_ci_public(r):
  72. failed_routes.append(r)
  73. if len(failed_routes):
  74. print("failed routes:", failed_routes)