dmonitoringmodeld.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. #!/usr/bin/env python3
  2. import os
  3. import gc
  4. import math
  5. import time
  6. import ctypes
  7. import numpy as np
  8. from pathlib import Path
  9. from setproctitle import setproctitle
  10. from cereal import messaging
  11. from cereal.messaging import PubMaster, SubMaster
  12. from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
  13. from openpilot.common.swaglog import cloudlog
  14. from openpilot.common.params import Params
  15. from openpilot.common.realtime import set_realtime_priority
  16. from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
  17. from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext
  18. from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid
  19. CALIB_LEN = 3
  20. MODEL_WIDTH = 1440
  21. MODEL_HEIGHT = 960
  22. FEATURE_LEN = 512
  23. OUTPUT_SIZE = 84 + FEATURE_LEN
  24. PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld"
  25. SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
  26. MODEL_PATHS = {
  27. ModelRunner.THNEED: Path(__file__).parent / 'models/dmonitoring_model.thneed',
  28. ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'}
  29. class DriverStateResult(ctypes.Structure):
  30. _fields_ = [
  31. ("face_orientation", ctypes.c_float*3),
  32. ("face_position", ctypes.c_float*3),
  33. ("face_orientation_std", ctypes.c_float*3),
  34. ("face_position_std", ctypes.c_float*3),
  35. ("face_prob", ctypes.c_float),
  36. ("_unused_a", ctypes.c_float*8),
  37. ("left_eye_prob", ctypes.c_float),
  38. ("_unused_b", ctypes.c_float*8),
  39. ("right_eye_prob", ctypes.c_float),
  40. ("left_blink_prob", ctypes.c_float),
  41. ("right_blink_prob", ctypes.c_float),
  42. ("sunglasses_prob", ctypes.c_float),
  43. ("occluded_prob", ctypes.c_float),
  44. ("ready_prob", ctypes.c_float*4),
  45. ("not_ready_prob", ctypes.c_float*2)]
  46. class DMonitoringModelResult(ctypes.Structure):
  47. _fields_ = [
  48. ("driver_state_lhd", DriverStateResult),
  49. ("driver_state_rhd", DriverStateResult),
  50. ("poor_vision_prob", ctypes.c_float),
  51. ("wheel_on_right_prob", ctypes.c_float),
  52. ("features", ctypes.c_float*FEATURE_LEN)]
  53. class ModelState:
  54. inputs: dict[str, np.ndarray]
  55. output: np.ndarray
  56. model: ModelRunner
  57. def __init__(self, cl_ctx):
  58. assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
  59. self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32)
  60. self.inputs = {
  61. 'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8),
  62. 'calib': np.zeros(CALIB_LEN, dtype=np.float32)}
  63. self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx)
  64. self.model.addInput("input_img", None)
  65. self.model.addInput("calib", self.inputs['calib'])
  66. def run(self, buf:VisionBuf, calib:np.ndarray) -> tuple[np.ndarray, float]:
  67. self.inputs['calib'][:] = calib
  68. v_offset = buf.height - MODEL_HEIGHT
  69. h_offset = (buf.width - MODEL_WIDTH) // 2
  70. buf_data = buf.data.reshape(-1, buf.stride)
  71. input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH)
  72. input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH]
  73. self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
  74. t1 = time.perf_counter()
  75. self.model.execute()
  76. t2 = time.perf_counter()
  77. return self.output, t2 - t1
  78. def fill_driver_state(msg, ds_result: DriverStateResult):
  79. msg.faceOrientation = list(ds_result.face_orientation)
  80. msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std]
  81. msg.facePosition = list(ds_result.face_position[:2])
  82. msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]]
  83. msg.faceProb = float(sigmoid(ds_result.face_prob))
  84. msg.leftEyeProb = float(sigmoid(ds_result.left_eye_prob))
  85. msg.rightEyeProb = float(sigmoid(ds_result.right_eye_prob))
  86. msg.leftBlinkProb = float(sigmoid(ds_result.left_blink_prob))
  87. msg.rightBlinkProb = float(sigmoid(ds_result.right_blink_prob))
  88. msg.sunglassesProb = float(sigmoid(ds_result.sunglasses_prob))
  89. msg.occludedProb = float(sigmoid(ds_result.occluded_prob))
  90. msg.readyProb = [float(sigmoid(x)) for x in ds_result.ready_prob]
  91. msg.notReadyProb = [float(sigmoid(x)) for x in ds_result.not_ready_prob]
  92. def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, gpu_execution_time: float):
  93. model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents
  94. msg = messaging.new_message('driverStateV2', valid=True)
  95. ds = msg.driverStateV2
  96. ds.frameId = frame_id
  97. ds.modelExecutionTime = execution_time
  98. ds.gpuExecutionTime = gpu_execution_time
  99. ds.poorVisionProb = float(sigmoid(model_result.poor_vision_prob))
  100. ds.wheelOnRightProb = float(sigmoid(model_result.wheel_on_right_prob))
  101. ds.rawPredictions = model_output.tobytes() if SEND_RAW_PRED else b''
  102. fill_driver_state(ds.leftDriverData, model_result.driver_state_lhd)
  103. fill_driver_state(ds.rightDriverData, model_result.driver_state_rhd)
  104. return msg
  105. def main():
  106. gc.disable()
  107. setproctitle(PROCESS_NAME)
  108. set_realtime_priority(1)
  109. cl_context = CLContext()
  110. model = ModelState(cl_context)
  111. cloudlog.warning("models loaded, dmonitoringmodeld starting")
  112. Params().put_bool("DmModelInitialized", True)
  113. cloudlog.warning("connecting to driver stream")
  114. vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context)
  115. while not vipc_client.connect(False):
  116. time.sleep(0.1)
  117. assert vipc_client.is_connected()
  118. cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}")
  119. sm = SubMaster(["liveCalibration"])
  120. pm = PubMaster(["driverStateV2"])
  121. calib = np.zeros(CALIB_LEN, dtype=np.float32)
  122. # last = 0
  123. while True:
  124. buf = vipc_client.recv()
  125. if buf is None:
  126. continue
  127. sm.update(0)
  128. if sm.updated["liveCalibration"]:
  129. calib[:] = np.array(sm["liveCalibration"].rpyCalib)
  130. t1 = time.perf_counter()
  131. model_output, gpu_execution_time = model.run(buf, calib)
  132. t2 = time.perf_counter()
  133. pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time))
  134. # print("dmonitoring process: %.2fms, from last %.2fms\n" % (t2 - t1, t1 - last))
  135. # last = t1
  136. if __name__ == "__main__":
  137. main()