PSNPE.hpp 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. // =============================================================================
  2. //
  3. // Copyright (c) 2019-2021 Qualcomm Technologies, Inc.
  4. // All Rights Reserved.
  5. // Confidential and Proprietary - Qualcomm Technologies, Inc.
  6. //
  7. // =============================================================================
  8. #ifndef PSNPE_HPP
  9. #define PSNPE_HPP
  10. #include <cstdlib>
  11. #include <functional>
  12. #include "SNPE/SNPE.hpp"
  13. #include "DlSystem/UserBufferMap.hpp"
  14. #include "DlContainer/IDlContainer.hpp"
  15. #include "DlSystem/DlEnums.hpp"
  16. #include "DlSystem/ZdlExportDefine.hpp"
  17. #include "UserBufferList.hpp"
  18. #include "RuntimeConfigList.hpp"
  19. #include "ApplicationBufferMap.hpp"
  20. namespace zdl
  21. {
  22. namespace PSNPE
  23. {
  24. /** @addtogroup c_plus_plus_apis C++
  25. @{ */
  26. /**
  27. *@ brief build snpe instance in serial or parallel
  28. *
  29. */
  30. enum ZDL_EXPORT BuildMode {
  31. SERIAL = 0,
  32. PARALLEL = 1
  33. };
  34. /**
  35. * @brief Input and output transmission mode
  36. */
  37. enum ZDL_EXPORT InputOutputTransmissionMode
  38. {
  39. sync = 0,
  40. outputAsync = 1,
  41. inputOutputAsync = 2
  42. };
  43. /**
  44. * @brief A structure representing parameters of callback function of Async Output mode
  45. */
  46. struct ZDL_EXPORT OutputAsyncCallbackParam
  47. {
  48. size_t dataIndex;
  49. bool executeStatus;
  50. std::string errorMsg;
  51. OutputAsyncCallbackParam(size_t _index,bool _status, const std::string& _errorMsg = std::string())
  52. : dataIndex(_index),executeStatus(_status), errorMsg(_errorMsg){};
  53. };
  54. /**
  55. * @brief A structure representing parameters of callback function of Async Input/Output mode
  56. */
  57. struct ZDL_EXPORT InputOutputAsyncCallbackParam
  58. {
  59. size_t dataIndex;
  60. const ApplicationBufferMap& outputMap;
  61. bool executeStatus;
  62. std::string errorMsg;
  63. InputOutputAsyncCallbackParam(size_t _index, const ApplicationBufferMap& output_map,bool _status,
  64. const std::string _ErrorMsg = std::string())
  65. : dataIndex(_index)
  66. , outputMap(output_map)
  67. , executeStatus(_status)
  68. , errorMsg(_ErrorMsg){};
  69. };
  70. /**
  71. * @brief This callback is called when the output data is ready, only use for Output Async mode
  72. */
  73. using OutputAsyncCallbackFunc = std::function<void(OutputAsyncCallbackParam)>;
  74. /**
  75. * @brief This callback is called when the output data is ready, only use for Output-Input Async mode
  76. */
  77. using InputOutputAsyncCallbackFunc = std::function<void(InputOutputAsyncCallbackParam)>;
  78. /**
  79. * @brief This callback is called when the input data is ready,only use for Output-Input Async mode
  80. */
  81. using InputOutputAsyncInputCallback = std::function<std::shared_ptr<ApplicationBufferMap>(const std::vector<std::string> &,
  82. const zdl::DlSystem::StringList &)>;
  83. /**
  84. * @brief .
  85. *
  86. * A structure PSNPE configuration
  87. *
  88. */
  89. struct ZDL_EXPORT BuildConfig final
  90. {
  91. BuildMode buildMode = BuildMode::SERIAL; ///< Specify build in serial mode or parallel mode
  92. zdl::DlContainer::IDlContainer* container;///< The opened container ptr
  93. zdl::DlSystem::StringList outputBufferNames;///< Specify the output layer name
  94. zdl::DlSystem::StringList outputTensors;///< Specify the output layer name
  95. RuntimeConfigList runtimeConfigList;///< The runtime config list for PSNPE, @see RuntimeConfig
  96. size_t inputThreadNumbers = 1;///< Specify the number of threads used in the execution phase to process input data, only used in inputOutputAsync mode
  97. size_t outputThreadNumbers = 1;///< Specify the number of threads used in the execution phase to process output data, only used in inputOutputAsync and outputAsync mode
  98. OutputAsyncCallbackFunc outputCallback;///< The callback to deal with output data ,only used in outputAsync mode
  99. InputOutputAsyncCallbackFunc inputOutputCallback;///< The callback to deal with output data ,only used in inputOutputAsync mode
  100. InputOutputAsyncInputCallback inputOutputInputCallback;///< The callback to deal with input data ,only used in inputOutputAsync mode
  101. InputOutputTransmissionMode inputOutputTransmissionMode = InputOutputTransmissionMode::sync;///< Specify execution mode
  102. zdl::DlSystem::ProfilingLevel_t profilingLevel = zdl::DlSystem::ProfilingLevel_t::OFF;///< Specify profiling level for Diaglog
  103. uint64_t encode[2] = {0, 0};
  104. bool enableInitCache = false;
  105. std::string platformOptions;
  106. std::string diaglogOutputDir = "./diaglogs/"; ///< Specify a diaglog output directory to save the generated Diaglog files.
  107. };
  108. /**
  109. * @brief .
  110. *
  111. * The class for executing SNPE instances in parallel.
  112. */
  113. class ZDL_EXPORT PSNPE final
  114. {
  115. public:
  116. ~PSNPE();
  117. explicit PSNPE() noexcept :m_TransmissionMode(InputOutputTransmissionMode::sync){};
  118. /**
  119. * @brief Build snpe instances.
  120. *
  121. */
  122. bool build(BuildConfig& buildConfig) noexcept;
  123. /**
  124. * @brief Execute snpe instances in Async Output mode and Sync mode
  125. *
  126. * @param[in] inputBufferList A list of user buffers that contains the input data
  127. *
  128. * @param[in,out] outputBufferList A list of user buffers that will hold the output data
  129. *
  130. */
  131. bool execute(UserBufferList& inputBufferList, UserBufferList& outputBufferList) noexcept;
  132. /**
  133. * @brief Execute snpe instances in Async Input/Output mode
  134. *
  135. * @param[in]inputMap A map of input buffers that contains input data. The names of buffers
  136. * need to be matched with names retrived through getInputTensorNames()
  137. *
  138. * @param dataIndex Index of the input data
  139. *
  140. * @param isTF8buff Whether prefer to using 8 bit quantized element for inference
  141. *
  142. * @return True if executed successfully; flase, otherwise.
  143. */
  144. bool executeInputOutputAsync(const std::vector<std::string>& inputMap, size_t dataIndex, bool isTF8buff) noexcept;
  145. bool executeInputOutputAsync(const std::vector<std::string>& inputMap, size_t dataIndex, bool isTF8buff,bool isTF8Outputbuff) noexcept;
  146. /**
  147. * @brief Returns the input layer names of the network.
  148. *
  149. * @return StringList which contains the input layer names
  150. */
  151. const zdl::DlSystem::StringList getInputTensorNames() const noexcept;
  152. /**
  153. * @brief Returns the output layer names of the network.
  154. *
  155. * @return StringList which contains the output layer names
  156. */
  157. const zdl::DlSystem::StringList getOutputTensorNames() const noexcept;
  158. /**
  159. * @brief Returns the input tensor dimensions of the network.
  160. *
  161. * @return TensorShape which contains the dimensions.
  162. */
  163. const zdl::DlSystem::TensorShape getInputDimensions() const noexcept;
  164. const zdl::DlSystem::TensorShape getInputDimensions(const char *name) const noexcept;
  165. /**
  166. * @brief Returns attributes of buffers.
  167. *
  168. * @see zdl::SNPE
  169. *
  170. * @return BufferAttributes of input/output tensor named.
  171. */
  172. const zdl::DlSystem::TensorShape getBufferAttributesDims(const char *name) const noexcept;
  173. zdl::DlSystem::Optional<zdl::DlSystem::IBufferAttributes*> getInputOutputBufferAttributes(const char *name) const noexcept;
  174. bool registerIonBuffers(const zdl::DlSystem::UserMemoryMap& ionBufferMap) const noexcept;
  175. bool deregisterIonBuffers(const zdl::DlSystem::StringList& ionBufferNames) const noexcept;
  176. const char* getLastErrorString();
  177. private:
  178. PSNPE(const PSNPE&) = delete;
  179. PSNPE& operator=(const PSNPE&) = delete;
  180. zdl::PSNPE::InputOutputTransmissionMode m_TransmissionMode;
  181. };
  182. /** @} */ /* end_addtogroup c_plus_plus_apis C++ */
  183. } // namespace PSNPE
  184. } // namespace zdl
  185. #endif // PSNPE_HPP