navmodeld.py 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117
  1. #!/usr/bin/env python3
  2. import gc
  3. import math
  4. import time
  5. import ctypes
  6. import numpy as np
  7. from pathlib import Path
  8. from cereal import messaging
  9. from cereal.messaging import PubMaster, SubMaster
  10. from cereal.visionipc import VisionIpcClient, VisionStreamType
  11. from openpilot.common.swaglog import cloudlog
  12. from openpilot.common.params import Params
  13. from openpilot.common.realtime import set_realtime_priority
  14. from openpilot.selfdrive.modeld.constants import ModelConstants
  15. from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
  16. NAV_INPUT_SIZE = 256*256
  17. NAV_FEATURE_LEN = 256
  18. NAV_DESIRE_LEN = 32
  19. NAV_OUTPUT_SIZE = 2*2*ModelConstants.IDX_N + NAV_DESIRE_LEN + NAV_FEATURE_LEN
  20. MODEL_PATHS = {
  21. ModelRunner.SNPE: Path(__file__).parent / 'models/navmodel_q.dlc',
  22. ModelRunner.ONNX: Path(__file__).parent / 'models/navmodel.onnx'}
  23. class NavModelOutputXY(ctypes.Structure):
  24. _fields_ = [
  25. ("x", ctypes.c_float),
  26. ("y", ctypes.c_float)]
  27. class NavModelOutputPlan(ctypes.Structure):
  28. _fields_ = [
  29. ("mean", NavModelOutputXY*ModelConstants.IDX_N),
  30. ("std", NavModelOutputXY*ModelConstants.IDX_N)]
  31. class NavModelResult(ctypes.Structure):
  32. _fields_ = [
  33. ("plan", NavModelOutputPlan),
  34. ("desire_pred", ctypes.c_float*NAV_DESIRE_LEN),
  35. ("features", ctypes.c_float*NAV_FEATURE_LEN)]
  36. class ModelState:
  37. inputs: dict[str, np.ndarray]
  38. output: np.ndarray
  39. model: ModelRunner
  40. def __init__(self):
  41. assert ctypes.sizeof(NavModelResult) == NAV_OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
  42. self.output = np.zeros(NAV_OUTPUT_SIZE, dtype=np.float32)
  43. self.inputs = {'input_img': np.zeros(NAV_INPUT_SIZE, dtype=np.uint8)}
  44. self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None)
  45. self.model.addInput("input_img", None)
  46. def run(self, buf:np.ndarray) -> tuple[np.ndarray, float]:
  47. self.inputs['input_img'][:] = buf
  48. t1 = time.perf_counter()
  49. self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
  50. self.model.execute()
  51. t2 = time.perf_counter()
  52. return self.output, t2 - t1
  53. def get_navmodel_packet(model_output: np.ndarray, valid: bool, frame_id: int, location_ts: int, execution_time: float, dsp_execution_time: float):
  54. model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(NavModelResult)).contents
  55. msg = messaging.new_message('navModel')
  56. msg.valid = valid
  57. msg.navModel.frameId = frame_id
  58. msg.navModel.locationMonoTime = location_ts
  59. msg.navModel.modelExecutionTime = execution_time
  60. msg.navModel.dspExecutionTime = dsp_execution_time
  61. msg.navModel.features = model_result.features[:]
  62. msg.navModel.desirePrediction = model_result.desire_pred[:]
  63. msg.navModel.position.x = [p.x for p in model_result.plan.mean]
  64. msg.navModel.position.y = [p.y for p in model_result.plan.mean]
  65. msg.navModel.position.xStd = [math.exp(p.x) for p in model_result.plan.std]
  66. msg.navModel.position.yStd = [math.exp(p.y) for p in model_result.plan.std]
  67. return msg
  68. def main():
  69. gc.disable()
  70. set_realtime_priority(1)
  71. # there exists a race condition when two processes try to create a
  72. # SNPE model runner at the same time, wait for dmonitoringmodeld to finish
  73. cloudlog.warning("waiting for dmonitoringmodeld to initialize")
  74. if not Params().get_bool("DmModelInitialized", True):
  75. return
  76. model = ModelState()
  77. cloudlog.warning("models loaded, navmodeld starting")
  78. vipc_client = VisionIpcClient("navd", VisionStreamType.VISION_STREAM_MAP, True)
  79. while not vipc_client.connect(False):
  80. time.sleep(0.1)
  81. assert vipc_client.is_connected()
  82. cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}")
  83. sm = SubMaster(["navInstruction"])
  84. pm = PubMaster(["navModel"])
  85. while True:
  86. buf = vipc_client.recv()
  87. if buf is None:
  88. continue
  89. sm.update(0)
  90. t1 = time.perf_counter()
  91. model_output, dsp_execution_time = model.run(buf.data[:buf.uv_offset])
  92. t2 = time.perf_counter()
  93. valid = vipc_client.valid and sm.valid["navInstruction"]
  94. pm.send("navModel", get_navmodel_packet(model_output, valid, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, dsp_execution_time))
  95. if __name__ == "__main__":
  96. main()