map_renderer.cc 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. #include "selfdrive/navd/map_renderer.h"
  2. #include <cmath>
  3. #include <string>
  4. #include <QApplication>
  5. #include <QBuffer>
  6. #include "common/util.h"
  7. #include "common/timing.h"
  8. #include "common/swaglog.h"
  9. #include "selfdrive/ui/qt/maps/map_helpers.h"
  10. const float DEFAULT_ZOOM = 13.5; // Don't go below 13 or features will start to disappear
  11. const int HEIGHT = 512, WIDTH = 512;
  12. const int NUM_VIPC_BUFFERS = 4;
  13. const int EARTH_CIRCUMFERENCE_METERS = 40075000;
  14. const int PIXELS_PER_TILE = 256;
  15. const bool TEST_MODE = getenv("MAP_RENDER_TEST_MODE");
  16. const int LLK_DECIMATION = TEST_MODE ? 1 : 10;
  17. float get_zoom_level_for_scale(float lat, float meters_per_pixel) {
  18. float meters_per_tile = meters_per_pixel * PIXELS_PER_TILE;
  19. float num_tiles = cos(DEG2RAD(lat)) * EARTH_CIRCUMFERENCE_METERS / meters_per_tile;
  20. return log2(num_tiles) - 1;
  21. }
  22. MapRenderer::MapRenderer(const QMapboxGLSettings &settings, bool online) : m_settings(settings) {
  23. QSurfaceFormat fmt;
  24. fmt.setRenderableType(QSurfaceFormat::OpenGLES);
  25. ctx = std::make_unique<QOpenGLContext>();
  26. ctx->setFormat(fmt);
  27. ctx->create();
  28. assert(ctx->isValid());
  29. surface = std::make_unique<QOffscreenSurface>();
  30. surface->setFormat(ctx->format());
  31. surface->create();
  32. ctx->makeCurrent(surface.get());
  33. assert(QOpenGLContext::currentContext() == ctx.get());
  34. gl_functions.reset(ctx->functions());
  35. gl_functions->initializeOpenGLFunctions();
  36. QOpenGLFramebufferObjectFormat fbo_format;
  37. fbo.reset(new QOpenGLFramebufferObject(WIDTH, HEIGHT, fbo_format));
  38. std::string style = util::read_file(STYLE_PATH);
  39. m_map.reset(new QMapboxGL(nullptr, m_settings, fbo->size(), 1));
  40. m_map->setCoordinateZoom(QMapbox::Coordinate(0, 0), DEFAULT_ZOOM);
  41. m_map->setStyleJson(style.c_str());
  42. m_map->createRenderer();
  43. m_map->resize(fbo->size());
  44. m_map->setFramebufferObject(fbo->handle(), fbo->size());
  45. gl_functions->glViewport(0, 0, WIDTH, HEIGHT);
  46. QObject::connect(m_map.data(), &QMapboxGL::mapChanged, [=](QMapboxGL::MapChange change) {
  47. // https://github.com/mapbox/mapbox-gl-native/blob/cf734a2fec960025350d8de0d01ad38aeae155a0/platform/qt/include/qmapboxgl.hpp#L116
  48. //LOGD("new state %d", change);
  49. });
  50. QObject::connect(m_map.data(), &QMapboxGL::mapLoadingFailed, [=](QMapboxGL::MapLoadingFailure err_code, const QString &reason) {
  51. LOGE("Map loading failed with %d: '%s'\n", err_code, reason.toStdString().c_str());
  52. });
  53. if (online) {
  54. vipc_server.reset(new VisionIpcServer("navd"));
  55. vipc_server->create_buffers(VisionStreamType::VISION_STREAM_MAP, NUM_VIPC_BUFFERS, false, WIDTH/2, HEIGHT/2);
  56. vipc_server->start_listener();
  57. pm.reset(new PubMaster({"navThumbnail", "mapRenderState"}));
  58. sm.reset(new SubMaster({"liveLocationKalman", "navRoute"}, {"liveLocationKalman"}));
  59. timer = new QTimer(this);
  60. timer->setSingleShot(true);
  61. QObject::connect(timer, SIGNAL(timeout()), this, SLOT(msgUpdate()));
  62. timer->start(0);
  63. }
  64. }
  65. void MapRenderer::msgUpdate() {
  66. sm->update(1000);
  67. if (sm->updated("liveLocationKalman")) {
  68. auto location = (*sm)["liveLocationKalman"].getLiveLocationKalman();
  69. auto pos = location.getPositionGeodetic();
  70. auto orientation = location.getCalibratedOrientationNED();
  71. bool localizer_valid = (location.getStatus() == cereal::LiveLocationKalman::Status::VALID) && pos.getValid();
  72. if (localizer_valid && (sm->rcv_frame("liveLocationKalman") % LLK_DECIMATION) == 0) {
  73. updatePosition(QMapbox::Coordinate(pos.getValue()[0], pos.getValue()[1]), RAD2DEG(orientation.getValue()[2]));
  74. // TODO: use the static rendering mode
  75. if (!loaded() && frame_id > 0) {
  76. for (int i = 0; i < 5 && !loaded(); i++) {
  77. LOGW("map render retry #%d, %d", i+1, m_map.isNull());
  78. QApplication::processEvents(QEventLoop::AllEvents, 100);
  79. update();
  80. }
  81. if (!loaded()) {
  82. LOGE("failed to render map after retry");
  83. }
  84. }
  85. }
  86. }
  87. if (sm->updated("navRoute")) {
  88. QList<QGeoCoordinate> route;
  89. auto coords = (*sm)["navRoute"].getNavRoute().getCoordinates();
  90. for (auto const &c : coords) {
  91. route.push_back(QGeoCoordinate(c.getLatitude(), c.getLongitude()));
  92. }
  93. updateRoute(route);
  94. }
  95. // schedule next update
  96. timer->start(0);
  97. }
  98. void MapRenderer::updatePosition(QMapbox::Coordinate position, float bearing) {
  99. if (m_map.isNull()) {
  100. return;
  101. }
  102. // Choose a scale that ensures above 13 zoom level up to and above 75deg of lat
  103. float meters_per_pixel = 2;
  104. float zoom = get_zoom_level_for_scale(position.first, meters_per_pixel);
  105. m_map->setCoordinate(position);
  106. m_map->setBearing(bearing);
  107. m_map->setZoom(zoom);
  108. update();
  109. }
  110. bool MapRenderer::loaded() {
  111. return m_map->isFullyLoaded();
  112. }
  113. void MapRenderer::update() {
  114. double start_t = millis_since_boot();
  115. gl_functions->glClear(GL_COLOR_BUFFER_BIT);
  116. m_map->render();
  117. gl_functions->glFlush();
  118. double end_t = millis_since_boot();
  119. if ((vipc_server != nullptr) && loaded()) {
  120. publish((end_t - start_t) / 1000.0);
  121. }
  122. }
  123. void MapRenderer::sendThumbnail(const uint64_t ts, const kj::Array<capnp::byte> &buf) {
  124. MessageBuilder msg;
  125. auto thumbnaild = msg.initEvent().initNavThumbnail();
  126. thumbnaild.setFrameId(frame_id);
  127. thumbnaild.setTimestampEof(ts);
  128. thumbnaild.setThumbnail(buf);
  129. pm->send("navThumbnail", msg);
  130. }
  131. void MapRenderer::publish(const double render_time) {
  132. QImage cap = fbo->toImage().convertToFormat(QImage::Format_RGB888, Qt::AutoColor);
  133. uint64_t ts = nanos_since_boot();
  134. VisionBuf* buf = vipc_server->get_buffer(VisionStreamType::VISION_STREAM_MAP);
  135. VisionIpcBufExtra extra = {
  136. .frame_id = frame_id,
  137. .timestamp_sof = sm->rcv_time("liveLocationKalman"),
  138. .timestamp_eof = ts,
  139. };
  140. assert(cap.sizeInBytes() >= buf->len);
  141. uint8_t* dst = (uint8_t*)buf->addr;
  142. uint8_t* src = cap.bits();
  143. // RGB to greyscale and crop
  144. memset(dst, 128, buf->len);
  145. for (int r = 0; r < HEIGHT/2; r++) {
  146. for (int c = 0; c < WIDTH/2; c++) {
  147. dst[r*WIDTH/2 + c] = src[((HEIGHT/4 + r)*WIDTH + (c+WIDTH/4)) * 3];
  148. }
  149. }
  150. vipc_server->send(buf, &extra);
  151. // Send thumbnail
  152. if (TEST_MODE) {
  153. // Full image in thumbnails in test mode
  154. kj::Array<capnp::byte> buffer_kj = kj::heapArray<capnp::byte>((const capnp::byte*)cap.bits(), cap.sizeInBytes());
  155. sendThumbnail(ts, buffer_kj);
  156. } else if (frame_id % 100 == 0) {
  157. // Write jpeg into buffer
  158. QByteArray buffer_bytes;
  159. QBuffer buffer(&buffer_bytes);
  160. buffer.open(QIODevice::WriteOnly);
  161. cap.save(&buffer, "JPG", 50);
  162. kj::Array<capnp::byte> buffer_kj = kj::heapArray<capnp::byte>((const capnp::byte*)buffer_bytes.constData(), buffer_bytes.size());
  163. sendThumbnail(ts, buffer_kj);
  164. }
  165. // Send state msg
  166. MessageBuilder msg;
  167. auto state = msg.initEvent().initMapRenderState();
  168. state.setLocationMonoTime(sm->rcv_time("liveLocationKalman"));
  169. state.setRenderTime(render_time);
  170. state.setFrameId(frame_id);
  171. pm->send("mapRenderState", msg);
  172. frame_id++;
  173. }
  174. uint8_t* MapRenderer::getImage() {
  175. QImage cap = fbo->toImage().convertToFormat(QImage::Format_RGB888, Qt::AutoColor);
  176. uint8_t* src = cap.bits();
  177. uint8_t* dst = new uint8_t[WIDTH * HEIGHT];
  178. // RGB to greyscale
  179. for (int i = 0; i < WIDTH * HEIGHT; i++) {
  180. dst[i] = src[i * 3];
  181. }
  182. return dst;
  183. }
  184. void MapRenderer::updateRoute(QList<QGeoCoordinate> coordinates) {
  185. if (m_map.isNull()) return;
  186. initLayers();
  187. auto route_points = coordinate_list_to_collection(coordinates);
  188. QMapbox::Feature feature(QMapbox::Feature::LineStringType, route_points, {}, {});
  189. QVariantMap navSource;
  190. navSource["type"] = "geojson";
  191. navSource["data"] = QVariant::fromValue<QMapbox::Feature>(feature);
  192. m_map->updateSource("navSource", navSource);
  193. m_map->setLayoutProperty("navLayer", "visibility", "visible");
  194. }
  195. void MapRenderer::initLayers() {
  196. if (!m_map->layerExists("navLayer")) {
  197. QVariantMap nav;
  198. nav["id"] = "navLayer";
  199. nav["type"] = "line";
  200. nav["source"] = "navSource";
  201. m_map->addLayer(nav, "road-intersection");
  202. m_map->setPaintProperty("navLayer", "line-color", QColor("grey"));
  203. m_map->setPaintProperty("navLayer", "line-width", 5);
  204. m_map->setLayoutProperty("navLayer", "line-cap", "round");
  205. }
  206. }
  207. MapRenderer::~MapRenderer() {
  208. }
  209. extern "C" {
  210. MapRenderer* map_renderer_init(char *maps_host = nullptr, char *token = nullptr) {
  211. char *argv[] = {
  212. (char*)"navd",
  213. nullptr
  214. };
  215. int argc = 0;
  216. QApplication *app = new QApplication(argc, argv);
  217. assert(app);
  218. QMapboxGLSettings settings;
  219. settings.setApiBaseUrl(maps_host == nullptr ? MAPS_HOST : maps_host);
  220. settings.setAccessToken(token == nullptr ? get_mapbox_token() : token);
  221. return new MapRenderer(settings, false);
  222. }
  223. void map_renderer_update_position(MapRenderer *inst, float lat, float lon, float bearing) {
  224. inst->updatePosition({lat, lon}, bearing);
  225. QApplication::processEvents();
  226. }
  227. void map_renderer_update_route(MapRenderer *inst, char* polyline) {
  228. inst->updateRoute(polyline_to_coordinate_list(QString::fromUtf8(polyline)));
  229. }
  230. void map_renderer_update(MapRenderer *inst) {
  231. inst->update();
  232. }
  233. void map_renderer_process(MapRenderer *inst) {
  234. QApplication::processEvents();
  235. }
  236. bool map_renderer_loaded(MapRenderer *inst) {
  237. return inst->loaded();
  238. }
  239. uint8_t * map_renderer_get_image(MapRenderer *inst) {
  240. return inst->getImage();
  241. }
  242. void map_renderer_free_image(MapRenderer *inst, uint8_t * buf) {
  243. delete[] buf;
  244. }
  245. }