/src/ffmpegthumbs/ffmpegthumbnailer/moviedecoder.cpp
Line | Count | Source |
1 | | /* |
2 | | SPDX-FileCopyrightText: 2010 Dirk Vanden Boer <dirk.vdb@gmail.com> |
3 | | |
4 | | SPDX-License-Identifier: GPL-2.0-or-later |
5 | | */ |
6 | | |
7 | | #include "moviedecoder.h" |
8 | | #include "ffmpegthumbs_debug.h" |
9 | | |
10 | | #include <QDebug> |
11 | | #include <QFileInfo> |
12 | | |
13 | | extern "C" { |
14 | | #include <libswscale/swscale.h> |
15 | | #include <libavutil/display.h> |
16 | | #include <libavutil/imgutils.h> |
17 | | } |
18 | | |
19 | | using namespace std; |
20 | | |
21 | | namespace ffmpegthumbnailer |
22 | | { |
23 | | |
24 | | MovieDecoder::MovieDecoder(const QString& filename, AVFormatContext* pavContext) |
25 | 682 | : m_VideoStream(-1) |
26 | 682 | , m_pFormatContext(pavContext) |
27 | 682 | , m_pVideoCodecContext(nullptr) |
28 | 682 | , m_pVideoCodec(nullptr) |
29 | 682 | , m_pVideoStream(nullptr) |
30 | 682 | , m_pFrame(nullptr) |
31 | 682 | , m_pFrameBuffer(nullptr) |
32 | 682 | , m_pPacket(nullptr) |
33 | 682 | , m_FormatContextWasGiven(pavContext != nullptr) |
34 | 682 | , m_AllowSeek(true) |
35 | 682 | , m_initialized(false) |
36 | 682 | , m_bufferSinkContext(nullptr) |
37 | 682 | , m_bufferSourceContext(nullptr) |
38 | 682 | , m_filterGraph(nullptr) |
39 | 682 | , m_filterFrame(nullptr) |
40 | 682 | { |
41 | 682 | initialize(filename); |
42 | 682 | } |
43 | | |
44 | | MovieDecoder::~MovieDecoder() |
45 | 682 | { |
46 | 682 | destroy(); |
47 | 682 | } |
48 | | |
49 | | void MovieDecoder::initialize(const QString& filename) |
50 | 682 | { |
51 | 682 | m_lastWidth = -1; |
52 | 682 | m_lastHeight = -1; |
53 | 682 | m_lastPixfmt = AV_PIX_FMT_NONE; |
54 | | |
55 | | #if (LIBAVFORMAT_VERSION_MAJOR < 58) |
56 | | av_register_all(); |
57 | | #endif |
58 | | |
59 | 682 | QFileInfo fileInfo(filename); |
60 | | |
61 | 682 | if ((!m_FormatContextWasGiven) && avformat_open_input(&m_pFormatContext, fileInfo.absoluteFilePath().toLocal8Bit().data(), nullptr, nullptr) != 0) { |
62 | 682 | qCDebug(ffmpegthumbs_LOG) << "Could not open input file: " << fileInfo.absoluteFilePath(); |
63 | 682 | return; |
64 | 682 | } |
65 | | |
66 | 0 | if (avformat_find_stream_info(m_pFormatContext, nullptr) < 0) { |
67 | 0 | qCDebug(ffmpegthumbs_LOG) << "Could not find stream information"; |
68 | 0 | return; |
69 | 0 | } |
70 | | |
71 | 0 | if (!initializeVideo()) { |
72 | | // It already printed a message |
73 | 0 | return; |
74 | 0 | } |
75 | 0 | m_pFrame = av_frame_alloc(); |
76 | |
|
77 | 0 | if (m_pFrame) { |
78 | 0 | m_initialized=true; |
79 | 0 | } |
80 | 0 | } |
81 | | |
82 | | bool MovieDecoder::getInitialized() |
83 | 682 | { |
84 | 682 | return m_initialized; |
85 | 682 | } |
86 | | |
87 | | |
88 | | void MovieDecoder::destroy() |
89 | 682 | { |
90 | 682 | deleteFilterGraph(); |
91 | 682 | if (m_pVideoCodecContext) { |
92 | 0 | avcodec_free_context(&m_pVideoCodecContext); |
93 | 0 | m_pVideoCodecContext = nullptr; |
94 | 0 | } |
95 | 682 | m_pVideoStream = nullptr; |
96 | | |
97 | 682 | if ((!m_FormatContextWasGiven) && m_pFormatContext) { |
98 | 0 | avformat_close_input(&m_pFormatContext); |
99 | 0 | m_pFormatContext = nullptr; |
100 | 0 | } |
101 | | |
102 | 682 | if (m_pPacket) { |
103 | 0 | av_packet_unref(m_pPacket); |
104 | 0 | delete m_pPacket; |
105 | 0 | m_pPacket = nullptr; |
106 | 0 | } |
107 | | |
108 | 682 | if (m_pFrame) { |
109 | 0 | av_frame_free(&m_pFrame); |
110 | 0 | m_pFrame = nullptr; |
111 | 0 | } |
112 | | |
113 | 682 | if (m_pFrameBuffer) { |
114 | 0 | av_free(m_pFrameBuffer); |
115 | 0 | m_pFrameBuffer = nullptr; |
116 | 0 | } |
117 | 682 | } |
118 | | |
119 | | QString MovieDecoder::getCodec() |
120 | 0 | { |
121 | 0 | QString codecName; |
122 | 0 | if (m_pVideoCodec) { |
123 | 0 | codecName=QString::fromLatin1(m_pVideoCodec->name); |
124 | 0 | } |
125 | 0 | return codecName; |
126 | 0 | } |
127 | | |
128 | | bool MovieDecoder::initializeVideo() |
129 | 0 | { |
130 | 0 | m_VideoStream = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, &m_pVideoCodec, 0); |
131 | 0 | if (m_VideoStream < 0) { |
132 | 0 | qCDebug(ffmpegthumbs_LOG) << "Could not find video stream"; |
133 | 0 | return false; |
134 | 0 | } |
135 | | |
136 | 0 | m_pVideoCodecContext = avcodec_alloc_context3(m_pVideoCodec); |
137 | 0 | m_pVideoStream = m_pFormatContext->streams[m_VideoStream]; |
138 | 0 | avcodec_parameters_to_context(m_pVideoCodecContext, m_pVideoStream->codecpar); |
139 | |
|
140 | 0 | if (m_pVideoCodec == nullptr) { |
141 | | // set to nullptr, otherwise avcodec_close(m_pVideoCodecContext) crashes |
142 | 0 | m_pVideoCodecContext = nullptr; |
143 | 0 | qCDebug(ffmpegthumbs_LOG) << "Video Codec not found"; |
144 | 0 | return false; |
145 | 0 | } |
146 | | |
147 | 0 | m_pVideoCodecContext->workaround_bugs = 1; |
148 | |
|
149 | 0 | if (avcodec_open2(m_pVideoCodecContext, m_pVideoCodec, nullptr) < 0) { |
150 | 0 | qCDebug(ffmpegthumbs_LOG) << "Could not open video codec"; |
151 | 0 | return false; |
152 | 0 | } |
153 | | |
154 | 0 | return true; |
155 | 0 | } |
156 | | |
157 | | int MovieDecoder::getWidth() |
158 | 0 | { |
159 | 0 | if (m_pVideoCodecContext) { |
160 | 0 | return m_pVideoCodecContext->width; |
161 | 0 | } |
162 | | |
163 | 0 | return -1; |
164 | 0 | } |
165 | | |
166 | | int MovieDecoder::getHeight() |
167 | 0 | { |
168 | 0 | if (m_pVideoCodecContext) { |
169 | 0 | return m_pVideoCodecContext->height; |
170 | 0 | } |
171 | | |
172 | 0 | return -1; |
173 | 0 | } |
174 | | |
175 | | int MovieDecoder::getDuration() |
176 | 0 | { |
177 | 0 | if (m_pFormatContext) { |
178 | 0 | return static_cast<int>(m_pFormatContext->duration / AV_TIME_BASE); |
179 | 0 | } |
180 | | |
181 | 0 | return 0; |
182 | 0 | } |
183 | | |
184 | | void MovieDecoder::seek(int timeInSeconds) |
185 | 0 | { |
186 | 0 | if (!m_AllowSeek) { |
187 | 0 | return; |
188 | 0 | } |
189 | | |
190 | 0 | qint64 timestamp = AV_TIME_BASE * static_cast<qint64>(timeInSeconds); |
191 | |
|
192 | 0 | if (timestamp < 0) { |
193 | 0 | timestamp = 0; |
194 | 0 | } |
195 | |
|
196 | 0 | int ret = av_seek_frame(m_pFormatContext, -1, timestamp, 0); |
197 | 0 | if (ret >= 0) { |
198 | 0 | avcodec_flush_buffers(m_pVideoCodecContext); |
199 | 0 | } else { |
200 | 0 | qCDebug(ffmpegthumbs_LOG) << "Seeking in video failed"; |
201 | 0 | return; |
202 | 0 | } |
203 | | |
204 | 0 | int keyFrameAttempts = 0; |
205 | 0 | bool gotFrame = 0; |
206 | |
|
207 | 0 | do { |
208 | 0 | int count = 0; |
209 | 0 | gotFrame = 0; |
210 | |
|
211 | 0 | while (!gotFrame && count < 20) { |
212 | 0 | getVideoPacket(); |
213 | 0 | gotFrame = decodeVideoPacket(); |
214 | 0 | ++count; |
215 | 0 | } |
216 | |
|
217 | 0 | ++keyFrameAttempts; |
218 | 0 | } while ((!gotFrame || !(m_pFrame->flags & AV_FRAME_FLAG_KEY)) && keyFrameAttempts < 200); |
219 | |
|
220 | 0 | if (gotFrame == 0) { |
221 | 0 | qCDebug(ffmpegthumbs_LOG) << "Seeking in video failed"; |
222 | 0 | } |
223 | 0 | } |
224 | | |
225 | | |
226 | | bool MovieDecoder::decodeVideoFrame() |
227 | 0 | { |
228 | 0 | bool frameFinished = false; |
229 | |
|
230 | 0 | while (!frameFinished && getVideoPacket()) { |
231 | 0 | frameFinished = decodeVideoPacket(); |
232 | 0 | } |
233 | |
|
234 | 0 | if (!frameFinished) { |
235 | 0 | qCDebug(ffmpegthumbs_LOG) << "decodeVideoFrame() failed: frame not finished"; |
236 | 0 | } |
237 | |
|
238 | 0 | return frameFinished; |
239 | 0 | } |
240 | | |
241 | | bool MovieDecoder::decodeVideoPacket() |
242 | 0 | { |
243 | 0 | if (m_pPacket->stream_index != m_VideoStream) { |
244 | 0 | return false; |
245 | 0 | } |
246 | | |
247 | 0 | av_frame_unref(m_pFrame); |
248 | |
|
249 | 0 | avcodec_send_packet(m_pVideoCodecContext, m_pPacket); |
250 | 0 | int ret = avcodec_receive_frame(m_pVideoCodecContext, m_pFrame); |
251 | 0 | if (ret == AVERROR(EAGAIN)) { |
252 | 0 | return false; |
253 | 0 | } |
254 | | |
255 | 0 | return true; |
256 | 0 | } |
257 | | |
258 | | QImageIOHandler::Transformations MovieDecoder::transformations() |
259 | 0 | { |
260 | 0 | QImageIOHandler::Transformations ret = QImageIOHandler::TransformationNone; |
261 | 0 | if (!m_pVideoStream) { |
262 | 0 | qCWarning(ffmpegthumbs_LOG) << "No video stream!"; |
263 | 0 | return ret; |
264 | 0 | } |
265 | | |
266 | 0 | for (int i=0; i<m_pVideoStream->codecpar->nb_coded_side_data; i++) { |
267 | 0 | if (m_pVideoStream->codecpar->coded_side_data[i].type != AV_PKT_DATA_DISPLAYMATRIX) { |
268 | 0 | continue; |
269 | 0 | } |
270 | 0 | if (m_pVideoStream->codecpar->coded_side_data[i].size != sizeof(int32_t) * 9) { |
271 | 0 | qCWarning(ffmpegthumbs_LOG) << "Invalid display matrix size" << m_pVideoStream->codecpar->coded_side_data[i].size << "expected" << sizeof(int32_t) * 9; |
272 | 0 | continue; |
273 | 0 | } |
274 | 0 | int32_t *matrix = reinterpret_cast<int32_t*>(m_pVideoStream->codecpar->coded_side_data[i].data); |
275 | 0 | double rotation = av_display_rotation_get(matrix); |
276 | 0 | if (qFuzzyCompare(rotation, 0.)) { |
277 | 0 | ret |= QImageIOHandler::TransformationNone; |
278 | 0 | } else if (qFuzzyCompare(rotation, 90.)) { |
279 | 0 | ret |= QImageIOHandler::TransformationRotate270; |
280 | 0 | } else if (qFuzzyCompare(rotation, 180.) || qFuzzyCompare(rotation, -180.)) { |
281 | 0 | ret |= QImageIOHandler::TransformationRotate180; |
282 | 0 | } else if (qFuzzyCompare(rotation, -90.)) { |
283 | 0 | ret |= QImageIOHandler::TransformationRotate90; |
284 | 0 | } else { |
285 | 0 | qCWarning(ffmpegthumbs_LOG) << "Unhandled rotation" << rotation; |
286 | 0 | continue; |
287 | 0 | } |
288 | 0 | } |
289 | 0 | return ret; |
290 | 0 | } |
291 | | |
292 | | |
293 | | bool MovieDecoder::getVideoPacket() |
294 | 0 | { |
295 | 0 | bool framesAvailable = true; |
296 | 0 | bool frameDecoded = false; |
297 | |
|
298 | 0 | int attempts = 0; |
299 | |
|
300 | 0 | if (m_pPacket) { |
301 | 0 | av_packet_unref(m_pPacket); |
302 | 0 | delete m_pPacket; |
303 | 0 | } |
304 | |
|
305 | 0 | m_pPacket = new AVPacket(); |
306 | |
|
307 | 0 | while (framesAvailable && !frameDecoded && (attempts++ < 1000)) { |
308 | 0 | framesAvailable = av_read_frame(m_pFormatContext, m_pPacket) >= 0; |
309 | 0 | if (framesAvailable) { |
310 | 0 | frameDecoded = m_pPacket->stream_index == m_VideoStream; |
311 | 0 | if (!frameDecoded) { |
312 | 0 | av_packet_unref(m_pPacket); |
313 | 0 | } |
314 | 0 | } |
315 | 0 | } |
316 | |
|
317 | 0 | return frameDecoded; |
318 | 0 | } |
319 | | |
320 | | void MovieDecoder::deleteFilterGraph() |
321 | 682 | { |
322 | 682 | if (m_filterGraph) { |
323 | 0 | av_frame_free(&m_filterFrame); |
324 | 0 | avfilter_graph_free(&m_filterGraph); |
325 | 0 | m_filterGraph = nullptr; |
326 | 0 | } |
327 | 682 | } |
328 | | |
329 | | bool MovieDecoder::initFilterGraph(enum AVPixelFormat pixfmt, int width, int height) |
330 | 0 | { |
331 | 0 | AVFilterInOut *inputs = nullptr, *outputs = nullptr; |
332 | |
|
333 | 0 | deleteFilterGraph(); |
334 | 0 | m_filterGraph = avfilter_graph_alloc(); |
335 | |
|
336 | 0 | QByteArray arguments("buffer="); |
337 | 0 | arguments += "video_size=" + QByteArray::number(width) + 'x' + QByteArray::number(height) + ':'; |
338 | 0 | arguments += "pix_fmt=" + QByteArray::number(pixfmt) + ':'; |
339 | 0 | arguments += "time_base=1/1:pixel_aspect=0/1[in];"; |
340 | 0 | arguments += "[in]yadif[out];"; |
341 | 0 | arguments += "[out]buffersink"; |
342 | |
|
343 | 0 | int ret = avfilter_graph_parse2(m_filterGraph, arguments.constData(), &inputs, &outputs); |
344 | 0 | if (ret < 0) { |
345 | 0 | qCWarning(ffmpegthumbs_LOG) << "Unable to parse filter graph"; |
346 | 0 | return false; |
347 | 0 | } |
348 | | |
349 | 0 | if(inputs || outputs) |
350 | 0 | return -1; |
351 | | |
352 | 0 | ret = avfilter_graph_config(m_filterGraph, nullptr); |
353 | 0 | if (ret < 0) { |
354 | 0 | qCWarning(ffmpegthumbs_LOG) << "Unable to validate filter graph"; |
355 | 0 | return false; |
356 | 0 | } |
357 | | |
358 | 0 | m_bufferSourceContext = avfilter_graph_get_filter(m_filterGraph, "Parsed_buffer_0"); |
359 | 0 | m_bufferSinkContext = avfilter_graph_get_filter(m_filterGraph, "Parsed_buffersink_2"); |
360 | 0 | if (!m_bufferSourceContext || !m_bufferSinkContext) { |
361 | 0 | qCWarning(ffmpegthumbs_LOG) << "Unable to get source or sink"; |
362 | 0 | return false; |
363 | 0 | } |
364 | 0 | m_filterFrame = av_frame_alloc(); |
365 | 0 | m_lastWidth = width; |
366 | 0 | m_lastHeight = height; |
367 | 0 | m_lastPixfmt = pixfmt; |
368 | |
|
369 | 0 | return true; |
370 | 0 | } |
371 | | |
372 | | bool MovieDecoder::processFilterGraph(AVFrame *dst, const AVFrame *src, |
373 | | enum AVPixelFormat pixfmt, int width, int height) |
374 | 0 | { |
375 | 0 | if (!m_filterGraph || width != m_lastWidth || |
376 | 0 | height != m_lastHeight || pixfmt != m_lastPixfmt) { |
377 | |
|
378 | 0 | if (!initFilterGraph(pixfmt, width, height)) { |
379 | 0 | return false; |
380 | 0 | } |
381 | 0 | } |
382 | | |
383 | 0 | memcpy(m_filterFrame->data, src->data, sizeof(src->data)); |
384 | 0 | memcpy(m_filterFrame->linesize, src->linesize, sizeof(src->linesize)); |
385 | 0 | m_filterFrame->width = width; |
386 | 0 | m_filterFrame->height = height; |
387 | 0 | m_filterFrame->format = pixfmt; |
388 | |
|
389 | 0 | int ret = av_buffersrc_add_frame(m_bufferSourceContext, m_filterFrame); |
390 | 0 | if (ret < 0) { |
391 | 0 | return false; |
392 | 0 | } |
393 | | |
394 | 0 | ret = av_buffersink_get_frame(m_bufferSinkContext, m_filterFrame); |
395 | 0 | if (ret < 0) { |
396 | 0 | return false; |
397 | 0 | } |
398 | | |
399 | 0 | av_image_copy(dst->data, dst->linesize, (const uint8_t **)m_filterFrame->data, m_filterFrame->linesize, pixfmt, width, height); |
400 | 0 | av_frame_unref(m_filterFrame); |
401 | |
|
402 | 0 | return true; |
403 | 0 | } |
404 | | |
405 | | void MovieDecoder::getScaledVideoFrame(int scaledSize, bool maintainAspectRatio, VideoFrame& videoFrame) |
406 | 0 | { |
407 | 0 | if (m_pFrame->flags & AV_FRAME_FLAG_INTERLACED) { |
408 | 0 | processFilterGraph((AVFrame*) m_pFrame, (AVFrame*) m_pFrame, m_pVideoCodecContext->pix_fmt, |
409 | 0 | m_pVideoCodecContext->width, m_pVideoCodecContext->height); |
410 | 0 | } |
411 | |
|
412 | 0 | int scaledWidth, scaledHeight; |
413 | 0 | convertAndScaleFrame(AV_PIX_FMT_RGB24, scaledSize, maintainAspectRatio, scaledWidth, scaledHeight); |
414 | |
|
415 | 0 | videoFrame.width = scaledWidth; |
416 | 0 | videoFrame.height = scaledHeight; |
417 | 0 | videoFrame.lineSize = m_pFrame->linesize[0]; |
418 | |
|
419 | 0 | videoFrame.frameData.clear(); |
420 | 0 | const auto frameSize = videoFrame.lineSize * videoFrame.height; |
421 | 0 | if (frameSize < 1) { |
422 | 0 | return; // calling front() on an empty vector is undefined behavior; return empty frame data instead |
423 | 0 | } |
424 | 0 | videoFrame.frameData.resize(frameSize); |
425 | 0 | memcpy((&(videoFrame.frameData.front())), m_pFrame->data[0], frameSize); |
426 | 0 | } |
427 | | |
428 | | void MovieDecoder::convertAndScaleFrame(AVPixelFormat format, int scaledSize, bool maintainAspectRatio, int& scaledWidth, int& scaledHeight) |
429 | 0 | { |
430 | 0 | calculateDimensions(scaledSize, maintainAspectRatio, scaledWidth, scaledHeight); |
431 | 0 | SwsContext* scaleContext = sws_getContext(m_pVideoCodecContext->width, m_pVideoCodecContext->height, |
432 | 0 | m_pVideoCodecContext->pix_fmt, scaledWidth, scaledHeight, |
433 | 0 | format, SWS_BICUBIC, nullptr, nullptr, nullptr); |
434 | |
|
435 | 0 | if (nullptr == scaleContext) { |
436 | 0 | qCDebug(ffmpegthumbs_LOG) << "Failed to create resize context"; |
437 | 0 | return; |
438 | 0 | } |
439 | | |
440 | 0 | AVFrame* convertedFrame = nullptr; |
441 | 0 | uint8_t* convertedFrameBuffer = nullptr; |
442 | |
|
443 | 0 | createAVFrame(&convertedFrame, &convertedFrameBuffer, scaledWidth, scaledHeight, format); |
444 | |
|
445 | 0 | sws_scale(scaleContext, m_pFrame->data, m_pFrame->linesize, 0, m_pVideoCodecContext->height, |
446 | 0 | convertedFrame->data, convertedFrame->linesize); |
447 | 0 | sws_freeContext(scaleContext); |
448 | |
|
449 | 0 | av_frame_free(&m_pFrame); |
450 | 0 | av_free(m_pFrameBuffer); |
451 | |
|
452 | 0 | m_pFrame = convertedFrame; |
453 | 0 | m_pFrameBuffer = convertedFrameBuffer; |
454 | 0 | } |
455 | | |
456 | | void MovieDecoder::calculateDimensions(int squareSize, bool maintainAspectRatio, int& destWidth, int& destHeight) |
457 | 0 | { |
458 | 0 | if (!maintainAspectRatio) { |
459 | 0 | destWidth = squareSize; |
460 | 0 | destHeight = squareSize; |
461 | 0 | } else { |
462 | 0 | int srcWidth = m_pVideoCodecContext->width; |
463 | 0 | int srcHeight = m_pVideoCodecContext->height; |
464 | 0 | int ascpectNominator = m_pVideoCodecContext->sample_aspect_ratio.num; |
465 | 0 | int ascpectDenominator = m_pVideoCodecContext->sample_aspect_ratio.den; |
466 | |
|
467 | 0 | if (ascpectNominator != 0 && ascpectDenominator != 0) { |
468 | 0 | srcWidth = srcWidth * ascpectNominator / ascpectDenominator; |
469 | 0 | } |
470 | |
|
471 | 0 | if (srcWidth > srcHeight) { |
472 | 0 | destWidth = squareSize; |
473 | 0 | destHeight = static_cast<int>(static_cast<float>(squareSize) / srcWidth * srcHeight); |
474 | 0 | } else { |
475 | 0 | destWidth = static_cast<int>(static_cast<float>(squareSize) / srcHeight * srcWidth); |
476 | 0 | destHeight = squareSize; |
477 | 0 | } |
478 | 0 | } |
479 | 0 | } |
480 | | |
481 | | void MovieDecoder::createAVFrame(AVFrame** avFrame, quint8** frameBuffer, int width, int height, AVPixelFormat format) |
482 | 0 | { |
483 | 0 | *avFrame = av_frame_alloc(); |
484 | |
|
485 | 0 | int numBytes = av_image_get_buffer_size (format, width + 1, height + 1, 16); |
486 | 0 | *frameBuffer = reinterpret_cast<quint8*>(av_malloc(numBytes)); |
487 | 0 | av_image_fill_arrays ((*avFrame)->data, (*avFrame)->linesize, *frameBuffer, format, width, height, 1); |
488 | 0 | } |
489 | | |
490 | | } |