File indexing completed on 2024-04-28 08:41:13
0001 /* 0002 SPDX-FileCopyrightText: 2010 Dirk Vanden Boer <dirk.vdb@gmail.com> 0003 0004 SPDX-License-Identifier: GPL-2.0-or-later 0005 */ 0006 0007 #include "moviedecoder.h" 0008 #include "ffmpegthumbs_debug.h" 0009 0010 #include <QDebug> 0011 #include <QFileInfo> 0012 0013 extern "C" { 0014 #include <libswscale/swscale.h> 0015 #include <libavutil/display.h> 0016 #include <libavutil/imgutils.h> 0017 } 0018 0019 using namespace std; 0020 0021 namespace ffmpegthumbnailer 0022 { 0023 0024 MovieDecoder::MovieDecoder(const QString& filename, AVFormatContext* pavContext) 0025 : m_VideoStream(-1) 0026 , m_pFormatContext(pavContext) 0027 , m_pVideoCodecContext(nullptr) 0028 , m_pVideoCodec(nullptr) 0029 , m_pVideoStream(nullptr) 0030 , m_pFrame(nullptr) 0031 , m_pFrameBuffer(nullptr) 0032 , m_pPacket(nullptr) 0033 , m_FormatContextWasGiven(pavContext != nullptr) 0034 , m_AllowSeek(true) 0035 , m_initialized(false) 0036 , m_bufferSinkContext(nullptr) 0037 , m_bufferSourceContext(nullptr) 0038 , m_filterGraph(nullptr) 0039 , m_filterFrame(nullptr) 0040 { 0041 initialize(filename); 0042 } 0043 0044 MovieDecoder::~MovieDecoder() 0045 { 0046 destroy(); 0047 } 0048 0049 void MovieDecoder::initialize(const QString& filename) 0050 { 0051 m_lastWidth = -1; 0052 m_lastHeight = -1; 0053 m_lastPixfmt = AV_PIX_FMT_NONE; 0054 0055 #if (LIBAVFORMAT_VERSION_MAJOR < 58) 0056 av_register_all(); 0057 #endif 0058 0059 QFileInfo fileInfo(filename); 0060 0061 if ((!m_FormatContextWasGiven) && avformat_open_input(&m_pFormatContext, fileInfo.absoluteFilePath().toLocal8Bit().data(), nullptr, nullptr) != 0) { 0062 qCDebug(ffmpegthumbs_LOG) << "Could not open input file: " << fileInfo.absoluteFilePath(); 0063 return; 0064 } 0065 0066 if (avformat_find_stream_info(m_pFormatContext, nullptr) < 0) { 0067 qCDebug(ffmpegthumbs_LOG) << "Could not find stream information"; 0068 return; 0069 } 0070 0071 if (!initializeVideo()) { 0072 // It already printed a message 0073 return; 0074 } 0075 m_pFrame = av_frame_alloc(); 0076 0077 if (m_pFrame) { 0078 m_initialized=true; 0079 } 0080 } 0081 0082 bool MovieDecoder::getInitialized() 0083 { 0084 return m_initialized; 0085 } 0086 0087 0088 void MovieDecoder::destroy() 0089 { 0090 deleteFilterGraph(); 0091 if (m_pVideoCodecContext) { 0092 avcodec_close(m_pVideoCodecContext); 0093 m_pVideoCodecContext = nullptr; 0094 } 0095 m_pVideoStream = nullptr; 0096 0097 if ((!m_FormatContextWasGiven) && m_pFormatContext) { 0098 avformat_close_input(&m_pFormatContext); 0099 m_pFormatContext = nullptr; 0100 } 0101 0102 if (m_pPacket) { 0103 av_packet_unref(m_pPacket); 0104 delete m_pPacket; 0105 m_pPacket = nullptr; 0106 } 0107 0108 if (m_pFrame) { 0109 av_frame_free(&m_pFrame); 0110 m_pFrame = nullptr; 0111 } 0112 0113 if (m_pFrameBuffer) { 0114 av_free(m_pFrameBuffer); 0115 m_pFrameBuffer = nullptr; 0116 } 0117 } 0118 0119 QString MovieDecoder::getCodec() 0120 { 0121 QString codecName; 0122 if (m_pVideoCodec) { 0123 codecName=QString::fromLatin1(m_pVideoCodec->name); 0124 } 0125 return codecName; 0126 } 0127 0128 bool MovieDecoder::initializeVideo() 0129 { 0130 m_VideoStream = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, &m_pVideoCodec, 0); 0131 if (m_VideoStream < 0) { 0132 qCDebug(ffmpegthumbs_LOG) << "Could not find video stream"; 0133 return false; 0134 } 0135 0136 m_pVideoCodecContext = avcodec_alloc_context3(m_pVideoCodec); 0137 m_pVideoStream = m_pFormatContext->streams[m_VideoStream]; 0138 avcodec_parameters_to_context(m_pVideoCodecContext, m_pVideoStream->codecpar); 0139 0140 if (m_pVideoCodec == nullptr) { 0141 // set to nullptr, otherwise avcodec_close(m_pVideoCodecContext) crashes 0142 m_pVideoCodecContext = nullptr; 0143 qCDebug(ffmpegthumbs_LOG) << "Video Codec not found"; 0144 return false; 0145 } 0146 0147 m_pVideoCodecContext->workaround_bugs = 1; 0148 0149 if (avcodec_open2(m_pVideoCodecContext, m_pVideoCodec, nullptr) < 0) { 0150 qCDebug(ffmpegthumbs_LOG) << "Could not open video codec"; 0151 return false; 0152 } 0153 0154 return true; 0155 } 0156 0157 int MovieDecoder::getWidth() 0158 { 0159 if (m_pVideoCodecContext) { 0160 return m_pVideoCodecContext->width; 0161 } 0162 0163 return -1; 0164 } 0165 0166 int MovieDecoder::getHeight() 0167 { 0168 if (m_pVideoCodecContext) { 0169 return m_pVideoCodecContext->height; 0170 } 0171 0172 return -1; 0173 } 0174 0175 int MovieDecoder::getDuration() 0176 { 0177 if (m_pFormatContext) { 0178 return static_cast<int>(m_pFormatContext->duration / AV_TIME_BASE); 0179 } 0180 0181 return 0; 0182 } 0183 0184 void MovieDecoder::seek(int timeInSeconds) 0185 { 0186 if (!m_AllowSeek) { 0187 return; 0188 } 0189 0190 qint64 timestamp = AV_TIME_BASE * static_cast<qint64>(timeInSeconds); 0191 0192 if (timestamp < 0) { 0193 timestamp = 0; 0194 } 0195 0196 int ret = av_seek_frame(m_pFormatContext, -1, timestamp, 0); 0197 if (ret >= 0) { 0198 avcodec_flush_buffers(m_pVideoCodecContext); 0199 } else { 0200 qCDebug(ffmpegthumbs_LOG) << "Seeking in video failed"; 0201 return; 0202 } 0203 0204 int keyFrameAttempts = 0; 0205 bool gotFrame = 0; 0206 0207 do { 0208 int count = 0; 0209 gotFrame = 0; 0210 0211 while (!gotFrame && count < 20) { 0212 getVideoPacket(); 0213 gotFrame = decodeVideoPacket(); 0214 ++count; 0215 } 0216 0217 ++keyFrameAttempts; 0218 } while ((!gotFrame || !m_pFrame->key_frame) && keyFrameAttempts < 200); 0219 0220 if (gotFrame == 0) { 0221 qCDebug(ffmpegthumbs_LOG) << "Seeking in video failed"; 0222 } 0223 } 0224 0225 0226 bool MovieDecoder::decodeVideoFrame() 0227 { 0228 bool frameFinished = false; 0229 0230 while (!frameFinished && getVideoPacket()) { 0231 frameFinished = decodeVideoPacket(); 0232 } 0233 0234 if (!frameFinished) { 0235 qCDebug(ffmpegthumbs_LOG) << "decodeVideoFrame() failed: frame not finished"; 0236 } 0237 0238 return frameFinished; 0239 } 0240 0241 bool MovieDecoder::decodeVideoPacket() 0242 { 0243 if (m_pPacket->stream_index != m_VideoStream) { 0244 return false; 0245 } 0246 0247 av_frame_unref(m_pFrame); 0248 0249 avcodec_send_packet(m_pVideoCodecContext, m_pPacket); 0250 int ret = avcodec_receive_frame(m_pVideoCodecContext, m_pFrame); 0251 if (ret == AVERROR(EAGAIN)) { 0252 return false; 0253 } 0254 0255 return true; 0256 } 0257 0258 QImageIOHandler::Transformations MovieDecoder::transformations() 0259 { 0260 QImageIOHandler::Transformations ret = QImageIOHandler::TransformationNone; 0261 if (!m_pVideoStream) { 0262 qCWarning(ffmpegthumbs_LOG) << "No video stream!"; 0263 return ret; 0264 } 0265 0266 for (int i=0; i<m_pVideoStream->nb_side_data; i++) { 0267 if (m_pVideoStream->side_data[i].type != AV_PKT_DATA_DISPLAYMATRIX) { 0268 continue; 0269 } 0270 if (m_pVideoStream->side_data[i].size != sizeof(int32_t) * 9) { 0271 qCWarning(ffmpegthumbs_LOG) << "Invalid display matrix size" << m_pVideoStream->side_data[i].size << "expected" << sizeof(int32_t) * 9; 0272 continue; 0273 } 0274 int32_t *matrix = reinterpret_cast<int32_t*>(m_pVideoStream->side_data[i].data); 0275 double rotation = av_display_rotation_get(matrix); 0276 if (qFuzzyCompare(rotation, 0.)) { 0277 ret |= QImageIOHandler::TransformationNone; 0278 } else if (qFuzzyCompare(rotation, 90.)) { 0279 ret |= QImageIOHandler::TransformationRotate270; 0280 } else if (qFuzzyCompare(rotation, 180.) || qFuzzyCompare(rotation, -180.)) { 0281 ret |= QImageIOHandler::TransformationRotate180; 0282 } else if (qFuzzyCompare(rotation, -90.)) { 0283 ret |= QImageIOHandler::TransformationRotate90; 0284 } else { 0285 qCWarning(ffmpegthumbs_LOG) << "Unhandled rotation" << rotation; 0286 continue; 0287 } 0288 } 0289 return ret; 0290 } 0291 0292 0293 bool MovieDecoder::getVideoPacket() 0294 { 0295 bool framesAvailable = true; 0296 bool frameDecoded = false; 0297 0298 int attempts = 0; 0299 0300 if (m_pPacket) { 0301 av_packet_unref(m_pPacket); 0302 delete m_pPacket; 0303 } 0304 0305 m_pPacket = new AVPacket(); 0306 0307 while (framesAvailable && !frameDecoded && (attempts++ < 1000)) { 0308 framesAvailable = av_read_frame(m_pFormatContext, m_pPacket) >= 0; 0309 if (framesAvailable) { 0310 frameDecoded = m_pPacket->stream_index == m_VideoStream; 0311 if (!frameDecoded) { 0312 av_packet_unref(m_pPacket); 0313 } 0314 } 0315 } 0316 0317 return frameDecoded; 0318 } 0319 0320 void MovieDecoder::deleteFilterGraph() 0321 { 0322 if (m_filterGraph) { 0323 av_frame_free(&m_filterFrame); 0324 avfilter_graph_free(&m_filterGraph); 0325 m_filterGraph = nullptr; 0326 } 0327 } 0328 0329 bool MovieDecoder::initFilterGraph(enum AVPixelFormat pixfmt, int width, int height) 0330 { 0331 AVFilterInOut *inputs = nullptr, *outputs = nullptr; 0332 0333 deleteFilterGraph(); 0334 m_filterGraph = avfilter_graph_alloc(); 0335 0336 QByteArray arguments("buffer="); 0337 arguments += "video_size=" + QByteArray::number(width) + 'x' + QByteArray::number(height) + ':'; 0338 arguments += "pix_fmt=" + QByteArray::number(pixfmt) + ':'; 0339 arguments += "time_base=1/1:pixel_aspect=0/1[in];"; 0340 arguments += "[in]yadif[out];"; 0341 arguments += "[out]buffersink"; 0342 0343 int ret = avfilter_graph_parse2(m_filterGraph, arguments.constData(), &inputs, &outputs); 0344 if (ret < 0) { 0345 qCWarning(ffmpegthumbs_LOG) << "Unable to parse filter graph"; 0346 return false; 0347 } 0348 0349 if(inputs || outputs) 0350 return -1; 0351 0352 ret = avfilter_graph_config(m_filterGraph, nullptr); 0353 if (ret < 0) { 0354 qCWarning(ffmpegthumbs_LOG) << "Unable to validate filter graph"; 0355 return false; 0356 } 0357 0358 m_bufferSourceContext = avfilter_graph_get_filter(m_filterGraph, "Parsed_buffer_0"); 0359 m_bufferSinkContext = avfilter_graph_get_filter(m_filterGraph, "Parsed_buffersink_2"); 0360 if (!m_bufferSourceContext || !m_bufferSinkContext) { 0361 qCWarning(ffmpegthumbs_LOG) << "Unable to get source or sink"; 0362 return false; 0363 } 0364 m_filterFrame = av_frame_alloc(); 0365 m_lastWidth = width; 0366 m_lastHeight = height; 0367 m_lastPixfmt = pixfmt; 0368 0369 return true; 0370 } 0371 0372 bool MovieDecoder::processFilterGraph(AVFrame *dst, const AVFrame *src, 0373 enum AVPixelFormat pixfmt, int width, int height) 0374 { 0375 if (!m_filterGraph || width != m_lastWidth || 0376 height != m_lastHeight || pixfmt != m_lastPixfmt) { 0377 0378 if (!initFilterGraph(pixfmt, width, height)) { 0379 return false; 0380 } 0381 } 0382 0383 memcpy(m_filterFrame->data, src->data, sizeof(src->data)); 0384 memcpy(m_filterFrame->linesize, src->linesize, sizeof(src->linesize)); 0385 m_filterFrame->width = width; 0386 m_filterFrame->height = height; 0387 m_filterFrame->format = pixfmt; 0388 0389 int ret = av_buffersrc_add_frame(m_bufferSourceContext, m_filterFrame); 0390 if (ret < 0) { 0391 return false; 0392 } 0393 0394 ret = av_buffersink_get_frame(m_bufferSinkContext, m_filterFrame); 0395 if (ret < 0) { 0396 return false; 0397 } 0398 0399 av_image_copy(dst->data, dst->linesize, (const uint8_t **)m_filterFrame->data, m_filterFrame->linesize, pixfmt, width, height); 0400 av_frame_unref(m_filterFrame); 0401 0402 return true; 0403 } 0404 0405 void MovieDecoder::getScaledVideoFrame(int scaledSize, bool maintainAspectRatio, VideoFrame& videoFrame) 0406 { 0407 if (m_pFrame->interlaced_frame) { 0408 processFilterGraph((AVFrame*) m_pFrame, (AVFrame*) m_pFrame, m_pVideoCodecContext->pix_fmt, 0409 m_pVideoCodecContext->width, m_pVideoCodecContext->height); 0410 } 0411 0412 int scaledWidth, scaledHeight; 0413 convertAndScaleFrame(AV_PIX_FMT_RGB24, scaledSize, maintainAspectRatio, scaledWidth, scaledHeight); 0414 0415 videoFrame.width = scaledWidth; 0416 videoFrame.height = scaledHeight; 0417 videoFrame.lineSize = m_pFrame->linesize[0]; 0418 0419 videoFrame.frameData.clear(); 0420 const auto frameSize = videoFrame.lineSize * videoFrame.height; 0421 if (frameSize < 1) { 0422 return; // calling front() on an empty vector is undefined behavior; return empty frame data instead 0423 } 0424 videoFrame.frameData.resize(frameSize); 0425 memcpy((&(videoFrame.frameData.front())), m_pFrame->data[0], frameSize); 0426 } 0427 0428 void MovieDecoder::convertAndScaleFrame(AVPixelFormat format, int scaledSize, bool maintainAspectRatio, int& scaledWidth, int& scaledHeight) 0429 { 0430 calculateDimensions(scaledSize, maintainAspectRatio, scaledWidth, scaledHeight); 0431 SwsContext* scaleContext = sws_getContext(m_pVideoCodecContext->width, m_pVideoCodecContext->height, 0432 m_pVideoCodecContext->pix_fmt, scaledWidth, scaledHeight, 0433 format, SWS_BICUBIC, nullptr, nullptr, nullptr); 0434 0435 if (nullptr == scaleContext) { 0436 qCDebug(ffmpegthumbs_LOG) << "Failed to create resize context"; 0437 return; 0438 } 0439 0440 AVFrame* convertedFrame = nullptr; 0441 uint8_t* convertedFrameBuffer = nullptr; 0442 0443 createAVFrame(&convertedFrame, &convertedFrameBuffer, scaledWidth, scaledHeight, format); 0444 0445 sws_scale(scaleContext, m_pFrame->data, m_pFrame->linesize, 0, m_pVideoCodecContext->height, 0446 convertedFrame->data, convertedFrame->linesize); 0447 sws_freeContext(scaleContext); 0448 0449 av_frame_free(&m_pFrame); 0450 av_free(m_pFrameBuffer); 0451 0452 m_pFrame = convertedFrame; 0453 m_pFrameBuffer = convertedFrameBuffer; 0454 } 0455 0456 void MovieDecoder::calculateDimensions(int squareSize, bool maintainAspectRatio, int& destWidth, int& destHeight) 0457 { 0458 if (!maintainAspectRatio) { 0459 destWidth = squareSize; 0460 destHeight = squareSize; 0461 } else { 0462 int srcWidth = m_pVideoCodecContext->width; 0463 int srcHeight = m_pVideoCodecContext->height; 0464 int ascpectNominator = m_pVideoCodecContext->sample_aspect_ratio.num; 0465 int ascpectDenominator = m_pVideoCodecContext->sample_aspect_ratio.den; 0466 0467 if (ascpectNominator != 0 && ascpectDenominator != 0) { 0468 srcWidth = srcWidth * ascpectNominator / ascpectDenominator; 0469 } 0470 0471 if (srcWidth > srcHeight) { 0472 destWidth = squareSize; 0473 destHeight = static_cast<int>(static_cast<float>(squareSize) / srcWidth * srcHeight); 0474 } else { 0475 destWidth = static_cast<int>(static_cast<float>(squareSize) / srcHeight * srcWidth); 0476 destHeight = squareSize; 0477 } 0478 } 0479 } 0480 0481 void MovieDecoder::createAVFrame(AVFrame** avFrame, quint8** frameBuffer, int width, int height, AVPixelFormat format) 0482 { 0483 *avFrame = av_frame_alloc(); 0484 0485 int numBytes = av_image_get_buffer_size (format, width + 1, height + 1, 16); 0486 *frameBuffer = reinterpret_cast<quint8*>(av_malloc(numBytes)); 0487 av_image_fill_arrays ((*avFrame)->data, (*avFrame)->linesize, *frameBuffer, format, width, height, 1); 0488 } 0489 0490 }