video/QtAVPlayer/qavhwdevice_videotoolbox.mm

0001 /*********************************************************
0002  * Copyright (C) 2020, Val Doroshchuk <valbok@gmail.com> *
0003  *                                                       *
0004  * This file is part of QtAVPlayer.                      *
0005  * Free Qt Media Player based on FFmpeg.                 *
0006  *********************************************************/
0007
0008 #include "qavhwdevice_videotoolbox_p.h"
0009 #include "qavvideobuffer_gpu_p.h"
0010
0011 #import <CoreVideo/CoreVideo.h>
0012 #if defined(Q_OS_MACOS)
0013 #import <IOSurface/IOSurface.h>
0014 #else
0015 #import <IOSurface/IOSurfaceRef.h>
0016 #endif
0017 #import <Metal/Metal.h>
0018
0019 #include <QList>
0020 #include <QVariant>
0021 #include <QDebug>
0022
0023 QT_BEGIN_NAMESPACE
0024
0025 class QAVHWDevice_VideoToolboxPrivate
0026 {
0027 public:
0028     id<MTLDevice> device = nullptr;
0029     CVPixelBufferRef pbuf = nullptr;
0030 };
0031
0032 QAVHWDevice_VideoToolbox::QAVHWDevice_VideoToolbox()
0033     : d_ptr(new QAVHWDevice_VideoToolboxPrivate)
0034 {
0035 }
0036
0037 QAVHWDevice_VideoToolbox::~QAVHWDevice_VideoToolbox()
0038 {
0039     Q_D(QAVHWDevice_VideoToolbox);
0040     CVPixelBufferRelease(d->pbuf);
0041     [d->device release];
0042 }
0043
0044 AVPixelFormat QAVHWDevice_VideoToolbox::format() const
0045 {
0046     return AV_PIX_FMT_VIDEOTOOLBOX;
0047 }
0048
0049 AVHWDeviceType QAVHWDevice_VideoToolbox::type() const
0050 {
0051     return AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
0052 }
0053
0054 class VideoBuffer_MTL : public QAVVideoBuffer_GPU
0055 {
0056 public:
0057     VideoBuffer_MTL(QAVHWDevice_VideoToolboxPrivate *hw, const QAVVideoFrame &frame)
0058         : QAVVideoBuffer_GPU(frame)
0059         , m_hw(hw)
0060     {
0061     }
0062
0063     QAVVideoFrame::HandleType handleType() const override
0064     {
0065         return QAVVideoFrame::MTLTextureHandle;
0066     }
0067
0068     QVariant handle(QRhi */*rhi*/) const override
0069     {
0070         CVPixelBufferRelease(m_hw->pbuf);
0071         m_hw->pbuf = (CVPixelBufferRef)frame().frame()->data[3];
0072         CVPixelBufferRetain(m_hw->pbuf);
0073         QList<QVariant> textures = { 0, 0 };
0074
0075         if (!m_hw->pbuf)
0076             return textures;
0077
0078         if (CVPixelBufferGetDataSize(m_hw->pbuf) <= 0)
0079             return textures;
0080
0081         auto format = CVPixelBufferGetPixelFormatType(m_hw->pbuf);
0082         if (format != '420v') {
0083             qWarning() << "420v is supported only";
0084             return textures;
0085         }
0086
0087         if (!m_hw->device)
0088             m_hw->device = MTLCreateSystemDefaultDevice();
0089
0090         IOSurfaceRef surface = CVPixelBufferGetIOSurface(m_hw->pbuf);
0091         int planes = CVPixelBufferGetPlaneCount(m_hw->pbuf);
0092         for (int i = 0; i < planes; ++i) {
0093             int w = IOSurfaceGetWidthOfPlane(surface, i);
0094             int h = IOSurfaceGetHeightOfPlane(surface, i) ;
0095             MTLPixelFormat f = i ?  MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm;
0096             MTLTextureDescriptor *desc = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:f width:w height:h mipmapped:NO];
0097
0098             textures[i] = quint64([m_hw->device newTextureWithDescriptor:desc iosurface:surface plane:i]);
0099         }
0100
0101         return textures;
0102     }
0103
0104     QAVHWDevice_VideoToolboxPrivate *m_hw = nullptr;
0105 };
0106
0107 QAVVideoBuffer *QAVHWDevice_VideoToolbox::videoBuffer(const QAVVideoFrame &frame) const
0108 {
0109     return new VideoBuffer_MTL(d_ptr.get(), frame);
0110 }
0111
0112 QT_END_NAMESPACE