vnc_negotiate.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534
  1. /****************************************************************************
  2. * graphics/vnc/vnc_negotiate.c
  3. *
  4. * Copyright (C) 2016-2017 Gregory Nutt. All rights reserved.
  5. * Author: Gregory Nutt <gnutt@nuttx.org>
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions
  9. * are met:
  10. *
  11. * 1. Redistributions of source code must retain the above copyright
  12. * notice, this list of conditions and the following disclaimer.
  13. * 2. Redistributions in binary form must reproduce the above copyright
  14. * notice, this list of conditions and the following disclaimer in
  15. * the documentation and/or other materials provided with the
  16. * distribution.
  17. * 3. Neither the name NuttX nor the names of its contributors may be
  18. * used to endorse or promote products derived from this software
  19. * without specific prior written permission.
  20. *
  21. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24. * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  25. * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26. * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27. * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  28. * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29. * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30. * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31. * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32. * POSSIBILITY OF SUCH DAMAGE.
  33. *
  34. ****************************************************************************/
  35. /****************************************************************************
  36. * Included Files
  37. ****************************************************************************/
  38. #include <nuttx/config.h>
  39. #include <stdint.h>
  40. #include <string.h>
  41. #include <errno.h>
  42. #include <assert.h>
  43. #if defined(CONFIG_VNCSERVER_DEBUG) && !defined(CONFIG_DEBUG_GRAPHICS)
  44. # undef CONFIG_DEBUG_ERROR
  45. # undef CONFIG_DEBUG_WARN
  46. # undef CONFIG_DEBUG_INFO
  47. # undef CONFIG_DEBUG_GRAPHICS_ERROR
  48. # undef CONFIG_DEBUG_GRAPHICS_WARN
  49. # undef CONFIG_DEBUG_GRAPHICS_INFO
  50. # define CONFIG_DEBUG_ERROR 1
  51. # define CONFIG_DEBUG_WARN 1
  52. # define CONFIG_DEBUG_INFO 1
  53. # define CONFIG_DEBUG_GRAPHICS 1
  54. # define CONFIG_DEBUG_GRAPHICS_ERROR 1
  55. # define CONFIG_DEBUG_GRAPHICS_WARN 1
  56. # define CONFIG_DEBUG_GRAPHICS_INFO 1
  57. #endif
  58. #include <debug.h>
  59. #ifdef CONFIG_NET_SOCKOPTS
  60. # include <sys/time.h>
  61. #endif
  62. #include <nuttx/video/fb.h>
  63. #include <nuttx/video/rfb.h>
  64. #include "vnc_server.h"
  65. /****************************************************************************
  66. * Private Data
  67. ****************************************************************************/
  68. #if defined(CONFIG_VNCSERVER_PROTO3p3)
  69. static const char g_vncproto[] = RFB_PROTOCOL_VERSION_3p3;
  70. #elif defined(CONFIG_VNCSERVER_PROTO3p8)
  71. static const char g_vncproto[] = RFB_PROTOCOL_VERSION_3p8;
  72. static const char g_nosecurity[] = "No security types are supported";
  73. #endif
  74. static const char g_vncname[] = CONFIG_VNCSERVER_NAME;
  75. /****************************************************************************
  76. * Public Functions
  77. ****************************************************************************/
  78. /****************************************************************************
  79. * Name: vnc_negotiate
  80. *
  81. * Description:
  82. * Perform the VNC initialization sequence after the client has successfully
  83. * connected to the server. Negotiate security, framebuffer and color
  84. * properties.
  85. *
  86. * Input Parameters:
  87. * session - An instance of the session structure.
  88. *
  89. * Returned Value:
  90. * Returns zero (OK) on success; a negated errno value on failure.
  91. *
  92. ****************************************************************************/
  93. int vnc_negotiate(FAR struct vnc_session_s *session)
  94. {
  95. #ifdef CONFIG_VNCSERVER_PROTO3p3
  96. FAR struct rfb_sectype_s *sectype;
  97. #else /* ifdef CONFIG_VNCSERVER_PROTO3p8 */
  98. FAR struct rfb_supported_sectypes_s *sectypes;
  99. FAR struct rfb_selected_sectype_s *sectype;
  100. FAR struct rfb_sectype_result_s *secresult;
  101. FAR struct rfb_sectype_fail_s *secfail;
  102. #endif
  103. FAR struct rfb_serverinit_s *serverinit;
  104. FAR struct rfb_pixelfmt_s *pixelfmt;
  105. FAR struct rfb_setpixelformat_s *setformat;
  106. FAR struct rfb_setencodings_s *encodings;
  107. ssize_t nsent;
  108. ssize_t nrecvd;
  109. size_t len;
  110. #ifdef CONFIG_NET_SOCKOPTS
  111. struct timeval tv;
  112. int ret;
  113. /* Set a receive timeout so that we don't hang if the client does not
  114. * respond according to RFB 3.3 protocol.
  115. */
  116. tv.tv_sec = 5;
  117. tv.tv_usec = 0;
  118. ret = psock_setsockopt(&session->connect, SOL_SOCKET, SO_RCVTIMEO,
  119. &tv, sizeof(struct timeval));
  120. if (ret < 0)
  121. {
  122. gerr("ERROR: Failed to set receive timeout: %d\n", ret);
  123. return ret;
  124. }
  125. #endif
  126. /* Inform the client of the VNC protocol version */
  127. ginfo("Send protocol version: %s\n", g_vncproto);
  128. len = strlen(g_vncproto);
  129. nsent = psock_send(&session->connect, g_vncproto, len, 0);
  130. if (nsent < 0)
  131. {
  132. gerr("ERROR: Send ProtocolVersion failed: %d\n", (int)nsent);
  133. return (int)nsent;
  134. }
  135. DEBUGASSERT(nsent == len);
  136. /* Receive the echo of the protocol string */
  137. ginfo("Receive echo from VNC client\n");
  138. nrecvd = psock_recv(&session->connect, session->inbuf, len, 0);
  139. if (nrecvd < 0)
  140. {
  141. gerr("ERROR: Receive protocol confirmation failed: %d\n", (int)nrecvd);
  142. return (int)nrecvd;
  143. }
  144. else if (nrecvd == 0)
  145. {
  146. gwarn("WARNING: Connection closed\n");
  147. return -ECONNABORTED;
  148. }
  149. DEBUGASSERT(nrecvd == len);
  150. #ifdef CONFIG_VNCSERVER_PROTO3p3
  151. /* Version 3.3: The server decides the security type and sends a single
  152. * word containing the security type: Tell the client that we won't use
  153. * any stinkin' security.
  154. */
  155. ginfo("Send SecurityType\n");
  156. sectype = (FAR struct rfb_sectype_s *)session->outbuf;
  157. rfb_putbe32(sectype->type, RFB_SECTYPE_NONE);
  158. nsent = psock_send(&session->connect, sectype,
  159. sizeof(struct rfb_sectype_s), 0);
  160. if (nsent < 0)
  161. {
  162. gerr("ERROR: Send Security failed: %d\n", (int)nsent);
  163. return (int)nsent;
  164. }
  165. DEBUGASSERT(nsent == sizeof(struct rfb_sectype_s));
  166. #else /* ifdef CONFIG_VNCSERVER_PROTO3p8 */
  167. /* Version 3.8: Offer the client a choice of security -- where None is the
  168. * only option offered.
  169. */
  170. ginfo("Send SupportedSecurityTypes\n");
  171. sectypes = (FAR struct rfb_supported_sectypes_s *)session->outbuf;
  172. sectypes->ntypes = 1;
  173. sectypes->type[0] = RFB_SECTYPE_NONE;
  174. nsent = psock_send(&session->connect, sectypes,
  175. SIZEOF_RFB_SUPPORTED_SECTYPES_S(1), 0);
  176. if (nsent < 0)
  177. {
  178. gerr("ERROR: Send SupportedSecurityTypes failed: %d\n", (int)nsent);
  179. return (int)nsent;
  180. }
  181. DEBUGASSERT(nsent == SIZEOF_RFB_SUPPORTED_SECTYPES_S(1));
  182. /* If the server listed at least one valid security type supported by the
  183. * client, the client sends back a single byte indicating which security
  184. * type is to be used on the connection.
  185. */
  186. ginfo("Receive SecurityType\n");
  187. sectype = (FAR struct rfb_selected_sectype_s *)session->inbuf;
  188. nrecvd = psock_recv(&session->connect, sectype,
  189. sizeof(struct rfb_selected_sectype_s), 0);
  190. if (nrecvd < 0)
  191. {
  192. gerr("ERROR: Receive SecurityType failed: %d\n", (int)nrecvd);
  193. return (int)nrecvd;
  194. }
  195. else if (nrecvd == 0)
  196. {
  197. gwarn("WARNING: Connection closed\n");
  198. return -ECONNABORTED;
  199. }
  200. DEBUGASSERT(nrecvd == sizeof(struct rfb_selected_sectype_s));
  201. ginfo("Send SecurityResult\n");
  202. secresult = (FAR struct rfb_sectype_result_s *)session->outbuf;
  203. if (sectype->type != RFB_SECTYPE_NONE)
  204. {
  205. gerr("ERROR: Received unsupported SecurityType: %d\n", sectype->type);
  206. /* REVISIT: Should send the reason string here */
  207. rfb_putbe32(secresult->result, RFB_SECTYPE_FAIL);
  208. nsent = psock_send(&session->connect, secresult,
  209. sizeof(struct rfb_sectype_result_s), 0);
  210. if (nsent < 0)
  211. {
  212. gerr("ERROR: Send SecurityResult failed: %d\n", (int)nsent);
  213. return (int)nsent;
  214. }
  215. DEBUGASSERT(nsent == sizeof(struct rfb_sectype_result_s));
  216. ginfo("Send failure reason\n");
  217. secfail = (FAR struct rfb_sectype_fail_s *)session->outbuf;
  218. len = strlen(g_nosecurity);
  219. rfb_putbe32(secfail->len, len);
  220. memcpy(secfail->str, g_nosecurity, len);
  221. nsent = psock_send(&session->connect, secfail,
  222. SIZEOF_RFB_SECTYPE_FAIL_S(len), 0);
  223. if (nsent < 0)
  224. {
  225. gerr("ERROR: Send failure reason failed: %d\n", (int)nsent);
  226. return (int)nsent;
  227. }
  228. DEBUGASSERT(nsent == SIZEOF_RFB_SECTYPE_FAIL_S(len));
  229. return -EPROTONOSUPPORT;
  230. }
  231. rfb_putbe32(secresult->result, RFB_SECTYPE_SUCCESS);
  232. nsent = psock_send(&session->connect, secresult,
  233. sizeof(struct rfb_sectype_result_s), 0);
  234. if (nsent < 0)
  235. {
  236. gerr("ERROR: Send SecurityResult failed: %d\n", (int)nsent);
  237. return (int)nsent;
  238. }
  239. DEBUGASSERT(nsent == sizeof(struct rfb_sectype_result_s));
  240. #endif
  241. /* Receive the ClientInit message
  242. *
  243. * "Once the client and server are sure that they’re happy to talk to one
  244. * another using the agreed security type, the protocol passes to the
  245. * initialization phase. The client sends a ClientInit message followed
  246. * by the server sending a ServerInit message."
  247. *
  248. * In this implementation, the sharing flag is ignored.
  249. */
  250. ginfo("Receive ClientInit\n");
  251. nrecvd = psock_recv(&session->connect, session->inbuf,
  252. sizeof(struct rfb_clientinit_s), 0);
  253. if (nrecvd < 0)
  254. {
  255. gerr("ERROR: Receive ClientInit failed: %d\n", (int)nrecvd);
  256. return (int)nrecvd;
  257. }
  258. else if (nrecvd == 0)
  259. {
  260. gwarn("WARNING: Connection closed\n");
  261. return -ECONNABORTED;
  262. }
  263. DEBUGASSERT(nrecvd == sizeof(struct rfb_clientinit_s));
  264. /* Send the ServerInit message
  265. *
  266. * "After receiving the ClientInit message, the server sends a ServerInit
  267. * message. This tells the client the width and height of the server’s
  268. * framebuffer, its pixel format and the name associated with the desktop:"
  269. *
  270. * RealVNC client supports this resolutions:
  271. * Full (all availlable colors) - Max resolution of the platform (TrueColor)
  272. * Medium (256 colors) - 256 colors (Paletted)
  273. * Low (64 colors) - RGB8 2:2:2 (default, TrueColor)
  274. * Very Low (8 colors) - RGB3 1:1:1 (TrueColor)
  275. */
  276. ginfo("Send ServerInit\n");
  277. serverinit = (FAR struct rfb_serverinit_s *)session->outbuf;
  278. rfb_putbe16(serverinit->width, CONFIG_VNCSERVER_SCREENWIDTH);
  279. rfb_putbe16(serverinit->height, CONFIG_VNCSERVER_SCREENHEIGHT);
  280. pixelfmt = &serverinit->format;
  281. pixelfmt->bpp = RFB_BITSPERPIXEL;
  282. pixelfmt->depth = RFB_PIXELDEPTH;
  283. pixelfmt->bigendian = 0;
  284. pixelfmt->truecolor = RFB_TRUECOLOR;
  285. rfb_putbe16(pixelfmt->rmax, RFB_RMAX);
  286. rfb_putbe16(pixelfmt->gmax, RFB_GMAX);
  287. rfb_putbe16(pixelfmt->bmax, RFB_BMAX);
  288. pixelfmt->rshift = RFB_RSHIFT;
  289. pixelfmt->gshift = RFB_GSHIFT;
  290. pixelfmt->bshift = RFB_BSHIFT;
  291. len = strlen(g_vncname);
  292. rfb_putbe32(serverinit->namelen, len);
  293. memcpy(serverinit->name, g_vncname, len);
  294. nsent = psock_send(&session->connect, serverinit,
  295. SIZEOF_RFB_SERVERINIT_S(len), 0);
  296. if (nsent < 0)
  297. {
  298. gerr("ERROR: Send ServerInit failed: %d\n", (int)nsent);
  299. return (int)nsent;
  300. }
  301. DEBUGASSERT(nsent == SIZEOF_RFB_SERVERINIT_S(len));
  302. /* We now expect to receive the SetPixelFormat message from the client.
  303. * This may override some of our framebuffer settings.
  304. */
  305. ginfo("Receive SetPixelFormat\n");
  306. setformat = (FAR struct rfb_setpixelformat_s *)session->inbuf;
  307. nrecvd = psock_recv(&session->connect, setformat,
  308. sizeof(struct rfb_setpixelformat_s), 0);
  309. if (nrecvd < 0)
  310. {
  311. gerr("ERROR: Receive SetPixelFormat failed: %d\n", (int)nrecvd);
  312. return (int)nrecvd;
  313. }
  314. else if (nrecvd == 0)
  315. {
  316. gwarn("WARNING: Connection closed\n");
  317. return -ECONNABORTED;
  318. }
  319. else if (nrecvd != sizeof(struct rfb_setpixelformat_s))
  320. {
  321. /* Must not be a SetPixelFormat message? */
  322. gerr("ERROR: SetFormat wrong size: %d\n", (int)nrecvd);
  323. return -EPROTO;
  324. }
  325. else if (setformat->msgtype != RFB_SETPIXELFMT_MSG)
  326. {
  327. gerr("ERROR: Not a SetPixelFormat message: %d\n",
  328. (int)setformat->msgtype);
  329. return -EPROTO;
  330. }
  331. /* Instantiate the client pixel format, verifying that the client request
  332. * format is one that we can handle.
  333. */
  334. ret = vnc_client_pixelformat(session, &setformat->format);
  335. if (ret < 0)
  336. {
  337. /* We do not support this pixel format */
  338. gerr("ERROR: PixelFormat not supported\n");
  339. return ret;
  340. }
  341. /* Receive supported encoding types from client. */
  342. ginfo("Receive encoding types\n");
  343. encodings = (FAR struct rfb_setencodings_s *)session->inbuf;
  344. nrecvd = psock_recv(&session->connect, encodings,
  345. CONFIG_VNCSERVER_INBUFFER_SIZE, 0);
  346. if (nrecvd < 0)
  347. {
  348. gerr("ERROR: Receive SetEncodings failed: %d\n", (int)nrecvd);
  349. return (int)nrecvd;
  350. }
  351. else if (nrecvd == 0)
  352. {
  353. gwarn("WARNING: Connection closed\n");
  354. return -ECONNABORTED;
  355. }
  356. if (encodings->msgtype == RFB_SETENCODINGS_MSG)
  357. {
  358. DEBUGASSERT(nrecvd >= SIZEOF_RFB_SETENCODINGS_S(0));
  359. /* Pick out any mutually supported encodings */
  360. ret = vnc_client_encodings(session, encodings);
  361. if (ret < 0)
  362. {
  363. gerr("ERROR: vnc_set_encodings failed: %d\n", ret);
  364. return ret;
  365. }
  366. }
  367. session->state = VNCSERVER_CONFIGURED;
  368. return OK;
  369. }
  370. /****************************************************************************
  371. * Name: vnc_client_pixelformat
  372. *
  373. * Description:
  374. * A Client-to-Sever SetPixelFormat message has been received. We need to
  375. * immediately switch the output color format that we generate.
  376. *
  377. * Input Parameters:
  378. * session - An instance of the session structure.
  379. * pixelfmt - The pixel from from the received SetPixelFormat message
  380. *
  381. * Returned Value:
  382. * Returns zero (OK) on success; a negated errno value on failure.
  383. *
  384. ****************************************************************************/
  385. int vnc_client_pixelformat(FAR struct vnc_session_s *session,
  386. FAR struct rfb_pixelfmt_s *pixelfmt)
  387. {
  388. if (pixelfmt->truecolor == 0)
  389. {
  390. /* At present, we support only TrueColor formats */
  391. gerr("ERROR: No support for palette colors\n");
  392. return -ENOSYS;
  393. }
  394. if (pixelfmt->bpp == 8 && pixelfmt->depth == 6)
  395. {
  396. ginfo("Client pixel format: RGB8 2:2:2\n");
  397. session->colorfmt = FB_FMT_RGB8_222;
  398. session->bpp = 8;
  399. session->bigendian = false;
  400. }
  401. else if (pixelfmt->bpp == 8 && pixelfmt->depth == 8)
  402. {
  403. ginfo("Client pixel format: RGB8 3:3:2\n");
  404. session->colorfmt = FB_FMT_RGB8_332;
  405. session->bpp = 8;
  406. session->bigendian = false;
  407. }
  408. else if (pixelfmt->bpp == 16 && pixelfmt->depth == 15)
  409. {
  410. ginfo("Client pixel format: RGB16 5:5:5\n");
  411. session->colorfmt = FB_FMT_RGB16_555;
  412. session->bpp = 16;
  413. session->bigendian = (pixelfmt->bigendian != 0) ? true : false;
  414. }
  415. else if (pixelfmt->bpp == 16 && pixelfmt->depth == 16)
  416. {
  417. ginfo("Client pixel format: RGB16 5:6:5\n");
  418. session->colorfmt = FB_FMT_RGB16_565;
  419. session->bpp = 16;
  420. session->bigendian = (pixelfmt->bigendian != 0) ? true : false;
  421. }
  422. else if (pixelfmt->bpp == 32 && pixelfmt->depth == 24)
  423. {
  424. ginfo("Client pixel format: RGB32 8:8:8\n");
  425. session->colorfmt = FB_FMT_RGB32;
  426. session->bpp = 32;
  427. session->bigendian = (pixelfmt->bigendian != 0) ? true : false;
  428. }
  429. else if (pixelfmt->bpp == 32 && pixelfmt->depth == 32)
  430. {
  431. session->colorfmt = FB_FMT_RGB32;
  432. session->bpp = 32;
  433. session->bigendian = (pixelfmt->bigendian != 0) ? true : false;
  434. }
  435. else
  436. {
  437. /* We do not support any other conversions */
  438. gerr("ERROR: No support for this BPP=%d and depth=%d\n",
  439. pixelfmt->bpp, pixelfmt->depth);
  440. return -ENOSYS;
  441. }
  442. session->change = true;
  443. return OK;
  444. }