camera.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179
  1. import itertools
  2. import numpy as np
  3. from dataclasses import dataclass
  4. import openpilot.common.transformations.orientation as orient
  5. ## -- hardcoded hardware params --
  6. @dataclass(frozen=True)
  7. class CameraConfig:
  8. width: int
  9. height: int
  10. focal_length: float
  11. @property
  12. def size(self):
  13. return (self.width, self.height)
  14. @property
  15. def intrinsics(self):
  16. # aka 'K' aka camera_frame_from_view_frame
  17. return np.array([
  18. [self.focal_length, 0.0, float(self.width)/2],
  19. [0.0, self.focal_length, float(self.height)/2],
  20. [0.0, 0.0, 1.0]
  21. ])
  22. @property
  23. def intrinsics_inv(self):
  24. # aka 'K_inv' aka view_frame_from_camera_frame
  25. return np.linalg.inv(self.intrinsics)
  26. @dataclass(frozen=True)
  27. class _NoneCameraConfig(CameraConfig):
  28. width: int = 0
  29. height: int = 0
  30. focal_length: float = 0
  31. @dataclass(frozen=True)
  32. class DeviceCameraConfig:
  33. fcam: CameraConfig
  34. dcam: CameraConfig
  35. ecam: CameraConfig
  36. def all_cams(self):
  37. for cam in ['fcam', 'dcam', 'ecam']:
  38. if not isinstance(getattr(self, cam), _NoneCameraConfig):
  39. yield cam, getattr(self, cam)
  40. _ar_ox_fisheye = CameraConfig(1928, 1208, 567.0) # focal length probably wrong? magnification is not consistent across frame
  41. _os_fisheye = CameraConfig(2688 // 2, 1520 // 2, 567.0 / 4 * 3)
  42. _ar_ox_config = DeviceCameraConfig(CameraConfig(1928, 1208, 2648.0), _ar_ox_fisheye, _ar_ox_fisheye)
  43. _os_config = DeviceCameraConfig(CameraConfig(2688 // 2, 1520 // 2, 1522.0 * 3 / 4), _os_fisheye, _os_fisheye)
  44. _neo_config = DeviceCameraConfig(CameraConfig(1164, 874, 910.0), CameraConfig(816, 612, 650.0), _NoneCameraConfig())
  45. DEVICE_CAMERAS = {
  46. # A "device camera" is defined by a device type and sensor
  47. # sensor type was never set on eon/neo/two
  48. ("neo", "unknown"): _neo_config,
  49. # unknown here is AR0231, field was added with OX03C10 support
  50. ("tici", "unknown"): _ar_ox_config,
  51. # before deviceState.deviceType was set, assume tici AR config
  52. ("unknown", "ar0231"): _ar_ox_config,
  53. ("unknown", "ox03c10"): _ar_ox_config,
  54. # simulator (emulates a tici)
  55. ("pc", "unknown"): _ar_ox_config,
  56. }
  57. prods = itertools.product(('tici', 'tizi', 'mici'), (('ar0231', _ar_ox_config), ('ox03c10', _ar_ox_config), ('os04c10', _os_config)))
  58. DEVICE_CAMERAS.update({(d, c[0]): c[1] for d, c in prods})
  59. # device/mesh : x->forward, y-> right, z->down
  60. # view : x->right, y->down, z->forward
  61. device_frame_from_view_frame = np.array([
  62. [ 0., 0., 1.],
  63. [ 1., 0., 0.],
  64. [ 0., 1., 0.]
  65. ])
  66. view_frame_from_device_frame = device_frame_from_view_frame.T
  67. # aka 'extrinsic_matrix'
  68. # road : x->forward, y -> left, z->up
  69. def get_view_frame_from_road_frame(roll, pitch, yaw, height):
  70. device_from_road = orient.rot_from_euler([roll, pitch, yaw]).dot(np.diag([1, -1, -1]))
  71. view_from_road = view_frame_from_device_frame.dot(device_from_road)
  72. return np.hstack((view_from_road, [[0], [height], [0]]))
  73. # aka 'extrinsic_matrix'
  74. def get_view_frame_from_calib_frame(roll, pitch, yaw, height):
  75. device_from_calib= orient.rot_from_euler([roll, pitch, yaw])
  76. view_from_calib = view_frame_from_device_frame.dot(device_from_calib)
  77. return np.hstack((view_from_calib, [[0], [height], [0]]))
  78. def vp_from_ke(m):
  79. """
  80. Computes the vanishing point from the product of the intrinsic and extrinsic
  81. matrices C = KE.
  82. The vanishing point is defined as lim x->infinity C (x, 0, 0, 1).T
  83. """
  84. return (m[0, 0]/m[2, 0], m[1, 0]/m[2, 0])
  85. def roll_from_ke(m):
  86. # note: different from calibration.h/RollAnglefromKE: i think that one's just wrong
  87. return np.arctan2(-(m[1, 0] - m[1, 1] * m[2, 0] / m[2, 1]),
  88. -(m[0, 0] - m[0, 1] * m[2, 0] / m[2, 1]))
  89. def normalize(img_pts, intrinsics):
  90. # normalizes image coordinates
  91. # accepts single pt or array of pts
  92. intrinsics_inv = np.linalg.inv(intrinsics)
  93. img_pts = np.array(img_pts)
  94. input_shape = img_pts.shape
  95. img_pts = np.atleast_2d(img_pts)
  96. img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
  97. img_pts_normalized = img_pts.dot(intrinsics_inv.T)
  98. img_pts_normalized[(img_pts < 0).any(axis=1)] = np.nan
  99. return img_pts_normalized[:, :2].reshape(input_shape)
  100. def denormalize(img_pts, intrinsics, width=np.inf, height=np.inf):
  101. # denormalizes image coordinates
  102. # accepts single pt or array of pts
  103. img_pts = np.array(img_pts)
  104. input_shape = img_pts.shape
  105. img_pts = np.atleast_2d(img_pts)
  106. img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1), dtype=img_pts.dtype)))
  107. img_pts_denormalized = img_pts.dot(intrinsics.T)
  108. if np.isfinite(width):
  109. img_pts_denormalized[img_pts_denormalized[:, 0] > width] = np.nan
  110. img_pts_denormalized[img_pts_denormalized[:, 0] < 0] = np.nan
  111. if np.isfinite(height):
  112. img_pts_denormalized[img_pts_denormalized[:, 1] > height] = np.nan
  113. img_pts_denormalized[img_pts_denormalized[:, 1] < 0] = np.nan
  114. return img_pts_denormalized[:, :2].reshape(input_shape)
  115. def get_calib_from_vp(vp, intrinsics):
  116. vp_norm = normalize(vp, intrinsics)
  117. yaw_calib = np.arctan(vp_norm[0])
  118. pitch_calib = -np.arctan(vp_norm[1]*np.cos(yaw_calib))
  119. roll_calib = 0
  120. return roll_calib, pitch_calib, yaw_calib
  121. def device_from_ecef(pos_ecef, orientation_ecef, pt_ecef):
  122. # device from ecef frame
  123. # device frame is x -> forward, y-> right, z -> down
  124. # accepts single pt or array of pts
  125. input_shape = pt_ecef.shape
  126. pt_ecef = np.atleast_2d(pt_ecef)
  127. ecef_from_device_rot = orient.rotations_from_quats(orientation_ecef)
  128. device_from_ecef_rot = ecef_from_device_rot.T
  129. pt_ecef_rel = pt_ecef - pos_ecef
  130. pt_device = np.einsum('jk,ik->ij', device_from_ecef_rot, pt_ecef_rel)
  131. return pt_device.reshape(input_shape)
  132. def img_from_device(pt_device):
  133. # img coordinates from pts in device frame
  134. # first transforms to view frame, then to img coords
  135. # accepts single pt or array of pts
  136. input_shape = pt_device.shape
  137. pt_device = np.atleast_2d(pt_device)
  138. pt_view = np.einsum('jk,ik->ij', view_frame_from_device_frame, pt_device)
  139. # This function should never return negative depths
  140. pt_view[pt_view[:, 2] < 0] = np.nan
  141. pt_img = pt_view/pt_view[:, 2:3]
  142. return pt_img.reshape(input_shape)[:, :2]