framereader.cc 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. #include "tools/replay/framereader.h"
  2. #include <map>
  3. #include <memory>
  4. #include <tuple>
  5. #include <utility>
  6. #include "common/util.h"
  7. #include "third_party/libyuv/include/libyuv.h"
  8. #include "tools/replay/util.h"
  9. #ifdef __APPLE__
  10. #define HW_DEVICE_TYPE AV_HWDEVICE_TYPE_VIDEOTOOLBOX
  11. #define HW_PIX_FMT AV_PIX_FMT_VIDEOTOOLBOX
  12. #else
  13. #define HW_DEVICE_TYPE AV_HWDEVICE_TYPE_CUDA
  14. #define HW_PIX_FMT AV_PIX_FMT_CUDA
  15. #endif
  16. namespace {
  17. enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) {
  18. enum AVPixelFormat *hw_pix_fmt = reinterpret_cast<enum AVPixelFormat *>(ctx->opaque);
  19. for (const enum AVPixelFormat *p = pix_fmts; *p != -1; p++) {
  20. if (*p == *hw_pix_fmt) return *p;
  21. }
  22. rWarning("Please run replay with the --no-hw-decoder flag!");
  23. *hw_pix_fmt = AV_PIX_FMT_NONE;
  24. return AV_PIX_FMT_YUV420P;
  25. }
  26. struct DecoderManager {
  27. VideoDecoder *acquire(CameraType type, AVCodecParameters *codecpar, bool hw_decoder) {
  28. auto key = std::tuple(type, codecpar->width, codecpar->height);
  29. std::unique_lock lock(mutex_);
  30. if (auto it = decoders_.find(key); it != decoders_.end()) {
  31. return it->second.get();
  32. }
  33. auto decoder = std::make_unique<VideoDecoder>();
  34. if (!decoder->open(codecpar, hw_decoder)) {
  35. decoder.reset(nullptr);
  36. }
  37. decoders_[key] = std::move(decoder);
  38. return decoders_[key].get();
  39. }
  40. std::mutex mutex_;
  41. std::map<std::tuple<CameraType, int, int>, std::unique_ptr<VideoDecoder>> decoders_;
  42. };
  43. DecoderManager decoder_manager;
  44. } // namespace
  45. FrameReader::FrameReader() {
  46. av_log_set_level(AV_LOG_QUIET);
  47. }
  48. FrameReader::~FrameReader() {
  49. if (input_ctx) avformat_close_input(&input_ctx);
  50. }
  51. bool FrameReader::load(CameraType type, const std::string &url, bool no_hw_decoder, std::atomic<bool> *abort, bool local_cache, int chunk_size, int retries) {
  52. auto local_file_path = url.find("https://") == 0 ? cacheFilePath(url) : url;
  53. if (!util::file_exists(local_file_path)) {
  54. FileReader f(local_cache, chunk_size, retries);
  55. if (f.read(url, abort).empty()) {
  56. return false;
  57. }
  58. }
  59. return loadFromFile(type, local_file_path, no_hw_decoder, abort);
  60. }
  61. bool FrameReader::loadFromFile(CameraType type, const std::string &file, bool no_hw_decoder, std::atomic<bool> *abort) {
  62. if (avformat_open_input(&input_ctx, file.c_str(), nullptr, nullptr) != 0 ||
  63. avformat_find_stream_info(input_ctx, nullptr) < 0) {
  64. rError("Failed to open input file or find video stream");
  65. return false;
  66. }
  67. input_ctx->probesize = 10 * 1024 * 1024; // 10MB
  68. decoder_ = decoder_manager.acquire(type, input_ctx->streams[0]->codecpar, !no_hw_decoder);
  69. if (!decoder_) {
  70. return false;
  71. }
  72. width = decoder_->width;
  73. height = decoder_->height;
  74. AVPacket pkt;
  75. packets_info.reserve(60 * 20); // 20fps, one minute
  76. while (!(abort && *abort) && av_read_frame(input_ctx, &pkt) == 0) {
  77. packets_info.emplace_back(PacketInfo{.flags = pkt.flags, .pos = pkt.pos});
  78. av_packet_unref(&pkt);
  79. }
  80. avio_seek(input_ctx->pb, 0, SEEK_SET);
  81. return !packets_info.empty();
  82. }
  83. bool FrameReader::get(int idx, VisionBuf *buf) {
  84. if (!buf || idx < 0 || idx >= packets_info.size()) {
  85. return false;
  86. }
  87. return decoder_->decode(this, idx, buf);
  88. }
  89. // class VideoDecoder
  90. VideoDecoder::VideoDecoder() {
  91. av_frame_ = av_frame_alloc();
  92. hw_frame_ = av_frame_alloc();
  93. }
  94. VideoDecoder::~VideoDecoder() {
  95. if (hw_device_ctx) av_buffer_unref(&hw_device_ctx);
  96. if (decoder_ctx) avcodec_free_context(&decoder_ctx);
  97. av_frame_free(&av_frame_);
  98. av_frame_free(&hw_frame_);
  99. }
  100. bool VideoDecoder::open(AVCodecParameters *codecpar, bool hw_decoder) {
  101. const AVCodec *decoder = avcodec_find_decoder(codecpar->codec_id);
  102. if (!decoder) return false;
  103. decoder_ctx = avcodec_alloc_context3(decoder);
  104. if (!decoder_ctx || avcodec_parameters_to_context(decoder_ctx, codecpar) != 0) {
  105. rError("Failed to allocate or initialize codec context");
  106. return false;
  107. }
  108. width = (decoder_ctx->width + 3) & ~3;
  109. height = decoder_ctx->height;
  110. if (hw_decoder && !initHardwareDecoder(HW_DEVICE_TYPE)) {
  111. rWarning("No device with hardware decoder found. fallback to CPU decoding.");
  112. }
  113. if (avcodec_open2(decoder_ctx, decoder, nullptr) < 0) {
  114. rError("Failed to open codec");
  115. return false;
  116. }
  117. return true;
  118. }
  119. bool VideoDecoder::initHardwareDecoder(AVHWDeviceType hw_device_type) {
  120. const AVCodecHWConfig *config = nullptr;
  121. for (int i = 0; (config = avcodec_get_hw_config(decoder_ctx->codec, i)) != nullptr; i++) {
  122. if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == hw_device_type) {
  123. hw_pix_fmt = config->pix_fmt;
  124. break;
  125. }
  126. }
  127. if (!config) {
  128. rWarning("Hardware configuration not found");
  129. return false;
  130. }
  131. int ret = av_hwdevice_ctx_create(&hw_device_ctx, hw_device_type, nullptr, nullptr, 0);
  132. if (ret < 0) {
  133. hw_pix_fmt = AV_PIX_FMT_NONE;
  134. rWarning("Failed to create specified HW device %d.", ret);
  135. return false;
  136. }
  137. decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
  138. decoder_ctx->opaque = &hw_pix_fmt;
  139. decoder_ctx->get_format = get_hw_format;
  140. return true;
  141. }
  142. bool VideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
  143. int from_idx = idx;
  144. if (idx != reader->prev_idx + 1) {
  145. // seeking to the nearest key frame
  146. for (int i = idx; i >= 0; --i) {
  147. if (reader->packets_info[i].flags & AV_PKT_FLAG_KEY) {
  148. from_idx = i;
  149. break;
  150. }
  151. }
  152. avio_seek(reader->input_ctx->pb, reader->packets_info[from_idx].pos, SEEK_SET);
  153. }
  154. reader->prev_idx = idx;
  155. bool result = false;
  156. AVPacket pkt;
  157. for (int i = from_idx; i <= idx; ++i) {
  158. if (av_read_frame(reader->input_ctx, &pkt) == 0) {
  159. AVFrame *f = decodeFrame(&pkt);
  160. if (f && i == idx) {
  161. result = copyBuffer(f, buf);
  162. }
  163. av_packet_unref(&pkt);
  164. }
  165. }
  166. return result;
  167. }
  168. AVFrame *VideoDecoder::decodeFrame(AVPacket *pkt) {
  169. int ret = avcodec_send_packet(decoder_ctx, pkt);
  170. if (ret < 0) {
  171. rError("Error sending a packet for decoding: %d", ret);
  172. return nullptr;
  173. }
  174. ret = avcodec_receive_frame(decoder_ctx, av_frame_);
  175. if (ret != 0) {
  176. rError("avcodec_receive_frame error: %d", ret);
  177. return nullptr;
  178. }
  179. if (av_frame_->format == hw_pix_fmt && av_hwframe_transfer_data(hw_frame_, av_frame_, 0) < 0) {
  180. rError("error transferring frame data from GPU to CPU");
  181. return nullptr;
  182. }
  183. return (av_frame_->format == hw_pix_fmt) ? hw_frame_ : av_frame_;
  184. }
  185. bool VideoDecoder::copyBuffer(AVFrame *f, VisionBuf *buf) {
  186. if (hw_pix_fmt == HW_PIX_FMT) {
  187. for (int i = 0; i < height/2; i++) {
  188. memcpy(buf->y + (i*2 + 0)*buf->stride, f->data[0] + (i*2 + 0)*f->linesize[0], width);
  189. memcpy(buf->y + (i*2 + 1)*buf->stride, f->data[0] + (i*2 + 1)*f->linesize[0], width);
  190. memcpy(buf->uv + i*buf->stride, f->data[1] + i*f->linesize[1], width);
  191. }
  192. } else {
  193. libyuv::I420ToNV12(f->data[0], f->linesize[0],
  194. f->data[1], f->linesize[1],
  195. f->data[2], f->linesize[2],
  196. buf->y, buf->stride,
  197. buf->uv, buf->stride,
  198. width, height);
  199. }
  200. return true;
  201. }