Warning, /multimedia/kdenlive/src/monitor/metalvideowidget.mm is written in an unsupported language. File is not indexed.
0001 /*
0002 SPDX-FileCopyrightText: 2023 Meltytech, LLC
0003 SPDX-License-Identifier: GPL-3.0-or-later
0004 */
0005
0006 #include "metalvideowidget.h"
0007 #include "core.h"
0008 #include "profiles/profilemodel.hpp"
0009 #include <Metal/Metal.h>
0010
0011
0012 class MetalVideoRenderer : public QObject
0013 {
0014 Q_OBJECT
0015 public:
0016 int displayRulerHeight = 0;
0017 MetalVideoRenderer()
0018 {
0019 for (int i = 0; i < 3; ++i) {
0020 m_ubuf[i] = nil;
0021 m_texture[i] = nil;
0022 }
0023 m_vbuf = nil;
0024 m_vs.first = nil;
0025 m_vs.second = nil;
0026 m_fs.first = nil;
0027 m_fs.second = nil;
0028 }
0029
0030 ~MetalVideoRenderer()
0031 {
0032 qDebug("cleanup");
0033
0034 for (int i = 0; i < 3; i++) {
0035 [m_texture[i] release];
0036 [m_ubuf[i] release];
0037 }
0038 [m_vbuf release];
0039 [m_vs.first release];
0040 [m_vs.second release];
0041 [m_fs.first release];
0042 [m_fs.second release];
0043 }
0044
0045 void initialize(QQuickWindow *window)
0046 {
0047 qDebug("init");
0048 m_window = window;
0049
0050 QSGRendererInterface *rif = m_window->rendererInterface();
0051
0052 // We are not prepared for anything other than running with the RHI and its Metal backend.
0053 Q_ASSERT(rif->graphicsApi() == QSGRendererInterface::Metal);
0054
0055 m_device = (id<MTLDevice>) rif->getResource(m_window, QSGRendererInterface::DeviceResource);
0056 Q_ASSERT(m_device);
0057
0058 if (m_vert.isEmpty())
0059 prepareShader(VertexStage);
0060 if (m_frag.isEmpty())
0061 prepareShader(FragmentStage);
0062
0063 m_vbuf = [m_device newBufferWithLength: 16*sizeof(float) options: MTLResourceStorageModeShared];
0064
0065 for (int i = 0; i < m_window->graphicsStateInfo().framesInFlight && i < 3; ++i)
0066 m_ubuf[i] = [m_device newBufferWithLength: sizeof(int) options: MTLResourceStorageModeShared];
0067
0068 MTLVertexDescriptor *inputLayout = [MTLVertexDescriptor vertexDescriptor];
0069 inputLayout.attributes[0].format = MTLVertexFormatFloat4;
0070 inputLayout.attributes[0].offset = 0;
0071 inputLayout.attributes[0].bufferIndex = 1; // ubuf is 0, vbuf is 1
0072 inputLayout.layouts[1].stride = 4 * sizeof(float);
0073
0074 MTLRenderPipelineDescriptor *rpDesc = [[MTLRenderPipelineDescriptor alloc] init];
0075 rpDesc.vertexDescriptor = inputLayout;
0076
0077 m_vs = compileShader(m_vert, m_vertEntryPoint);
0078 rpDesc.vertexFunction = m_vs.first;
0079 m_fs = compileShader(m_frag, m_fragEntryPoint);
0080 rpDesc.fragmentFunction = m_fs.first;
0081
0082 rpDesc.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
0083 // if (m_device.depth24Stencil8PixelFormatSupported) {
0084 // rpDesc.depthAttachmentPixelFormat = MTLPixelFormatDepth24Unorm_Stencil8;
0085 // rpDesc.stencilAttachmentPixelFormat = MTLPixelFormatDepth24Unorm_Stencil8;
0086 // } else {
0087 // rpDesc.depthAttachmentPixelFormat = MTLPixelFormatDepth32Float_Stencil8;
0088 // rpDesc.stencilAttachmentPixelFormat = MTLPixelFormatDepth32Float_Stencil8;
0089 // }
0090
0091 NSError *err = nil;
0092 m_pipeline = [m_device newRenderPipelineStateWithDescriptor: rpDesc error: &err];
0093 if (!m_pipeline) {
0094 const QString msg = QString::fromNSString(err.localizedDescription);
0095 qFatal("Failed to create render pipeline state: %s", qPrintable(msg));
0096 }
0097 [rpDesc release];
0098 }
0099
0100 void render(const QSize& viewportSize, const QRectF& videoRect, const double devicePixelRatio,
0101 const double zoom, const QPoint& offset, const SharedFrame& sharedFrame)
0102 {
0103 const QQuickWindow::GraphicsStateInfo &stateInfo(m_window->graphicsStateInfo());
0104
0105 QSGRendererInterface *rif = m_window->rendererInterface();
0106 id<MTLRenderCommandEncoder> encoder = (id<MTLRenderCommandEncoder>) rif->getResource(
0107 m_window, QSGRendererInterface::CommandEncoderResource);
0108 Q_ASSERT(encoder);
0109
0110 // Provide vertices of triangle strip
0111 float width = videoRect.width() * devicePixelRatio / 2.0f;
0112 float height = videoRect.height() * devicePixelRatio / 2.0f;
0113 float vertexData[] = { // x,y plus u,v texture coordinates
0114 -width, height, 0.f, 0.f,
0115 width, height, 1.f, 0.f,
0116 -width, -height, 0.f, 1.f,
0117 width, -height, 1.f, 1.f
0118 };
0119
0120 // Setup an orthographic projection
0121 QMatrix4x4 modelView;
0122 width = viewportSize.width() * devicePixelRatio;
0123 height = viewportSize.height() * devicePixelRatio;
0124 modelView.scale(2.0f / width, 2.0f / height);
0125
0126 // Set model-view
0127 if (videoRect.width() > 0.0 && zoom > 0.0) {
0128 if (offset.x() || offset.y())
0129 modelView.translate(-offset.x() * devicePixelRatio,
0130 offset.y() * devicePixelRatio);
0131 modelView.scale(zoom, zoom);
0132 }
0133 for (int i = 0; i < 4; i++) {
0134 vertexData[4 * i] *= modelView(0, 0);
0135 vertexData[4 * i] += modelView(0, 3);
0136 vertexData[4 * i + 1] *= modelView(1, 1);
0137 vertexData[4 * i + 1] += modelView(1, 3);
0138 }
0139
0140 m_window->beginExternalCommands();
0141
0142 void *p = [m_vbuf contents];
0143 memcpy(p, vertexData, sizeof(vertexData));
0144
0145 p = [m_ubuf[stateInfo.currentFrameSlot] contents];
0146 int colorspace = pCore->getCurrentProfile()->colorspace();
0147 memcpy(p, &colorspace, sizeof(colorspace));
0148
0149 MTLViewport vp;
0150 vp.originX = 0;
0151 vp.originY = -displayRulerHeight;
0152 vp.width = width;
0153 vp.height = height;
0154 vp.znear = 0;
0155 vp.zfar = 1;
0156 [encoder setViewport: vp];
0157
0158 // (Re)create the textures
0159 int iwidth = sharedFrame.get_image_width();
0160 int iheight = sharedFrame.get_image_height();
0161 const uint8_t *image = sharedFrame.get_image(mlt_image_yuv420p);
0162 for (int i = 0; i < 3; i++) {
0163 [m_texture[i] release];
0164 }
0165 m_texture[0] = initTexture(image, iwidth, iheight);
0166 m_texture[1] = initTexture(image + iwidth * iheight, iwidth / 2, iheight / 2);
0167 m_texture[2] = initTexture(image + iwidth * iheight + iwidth / 2 * iheight / 2, iwidth / 2,
0168 iheight / 2);
0169 // Set the texture object. The AAPLTextureIndexBaseColor enum value corresponds
0170 /// to the 'colorMap' argument in the 'samplingShader' function because its
0171 // texture attribute qualifier also uses AAPLTextureIndexBaseColor for its index.
0172 for (NSUInteger i = 0; i < 3; i++) {
0173 [encoder setFragmentTexture:m_texture[i] atIndex:i];
0174 }
0175
0176
0177 [encoder setFragmentBuffer: m_ubuf[stateInfo.currentFrameSlot] offset: 0 atIndex: 0];
0178 [encoder setVertexBuffer: m_vbuf offset: 0 atIndex: 1];
0179 [encoder setRenderPipelineState: m_pipeline];
0180 [encoder drawPrimitives: MTLPrimitiveTypeTriangleStrip vertexStart: 0 vertexCount: 4 instanceCount: 1 baseInstance: 0];
0181
0182 m_window->endExternalCommands();
0183 }
0184
0185 id<MTLTexture> initTexture(const void *p, NSUInteger width, NSUInteger height)
0186 {
0187 MTLTextureDescriptor *textureDescriptor = [[MTLTextureDescriptor alloc] init];
0188
0189 // 8-bit unsigned normalized value (i.e. 0 maps to 0.0 and 255 maps to 1.0)
0190 textureDescriptor.pixelFormat = MTLPixelFormatR8Unorm;
0191 textureDescriptor.width = width;
0192 textureDescriptor.height = height;
0193 id<MTLTexture> texture = [m_device newTextureWithDescriptor:textureDescriptor];
0194 [textureDescriptor release];
0195
0196 MTLRegion region = {
0197 { 0, 0, 0 }, // MTLOrigin
0198 {width, height, 1} // MTLSize
0199 };
0200
0201 // Copy the bytes from the data object into the texture
0202 [texture replaceRegion:region mipmapLevel:0 withBytes:p bytesPerRow:width];
0203 return texture;
0204 }
0205
0206 private:
0207 enum Stage {
0208 VertexStage,
0209 FragmentStage
0210 };
0211 using FuncAndLib = QPair<id<MTLFunction>, id<MTLLibrary> >;
0212 QQuickWindow *m_window = nullptr;
0213 QByteArray m_vert;
0214 QByteArray m_vertEntryPoint;
0215 QByteArray m_frag;
0216 QByteArray m_fragEntryPoint;
0217 id<MTLDevice> m_device;
0218 id<MTLBuffer> m_vbuf;
0219 id<MTLBuffer> m_ubuf[3];
0220 FuncAndLib m_vs;
0221 FuncAndLib m_fs;
0222 id<MTLRenderPipelineState> m_pipeline;
0223 id<MTLTexture> m_texture[3];
0224
0225 void prepareShader(Stage stage)
0226 {
0227 if (stage == VertexStage) {
0228 m_vert ="#include <metal_stdlib>\n"
0229 "#include <simd/simd.h>\n"
0230 "using namespace metal;"
0231 "struct main0_out {"
0232 " float2 coords [[user(locn0)]];"
0233 " float4 vertices [[position]];"
0234 "};"
0235 "struct main0_in {"
0236 " float4 vertices [[attribute(0)]];"
0237 "};"
0238 "vertex main0_out main0(main0_in in [[stage_in]]) {"
0239 " main0_out out = {};"
0240 " out.vertices = vector_float4(in.vertices.xy, 0.0f, 1.0f);"
0241 " out.coords = in.vertices.zw;"
0242 " return out;"
0243 "}";
0244 Q_ASSERT(!m_vert.isEmpty());
0245 m_vertEntryPoint = QByteArrayLiteral("main0");
0246 } else {
0247 m_frag ="#include <metal_stdlib>\n"
0248 "#include <simd/simd.h>\n"
0249 "using namespace metal;"
0250 "struct buf {"
0251 " int colorspace;"
0252 "};"
0253 "struct main0_out {"
0254 " float4 fragColor [[color(0)]];"
0255 "};"
0256 "struct main0_in {"
0257 " float2 coords [[user(locn0)]];"
0258 "};"
0259 "fragment main0_out main0(main0_in in [[stage_in]], constant buf& ubuf [[buffer(0)]],"
0260 " texture2d<half> yTex [[texture(0)]],"
0261 " texture2d<half> uTex [[texture(1)]],"
0262 " texture2d<half> vTex [[texture(2)]]"
0263 " ) {"
0264 " main0_out out = {};"
0265 " constexpr sampler yuvSampler (mag_filter::linear, min_filter::linear);"
0266 " float3 yuv;"
0267 " yuv.x = yTex.sample(yuvSampler, in.coords).r - 16.0f/255.0f;"
0268 " yuv.y = uTex.sample(yuvSampler, in.coords).r - 128.0f/255.0f;"
0269 " yuv.z = vTex.sample(yuvSampler, in.coords).r - 128.0f/255.0f;"
0270 " float3x3 coefficients;"
0271 " if (ubuf.colorspace == 601) {"
0272 " coefficients = float3x3("
0273 " {1.1643f, 1.1643f, 1.1643f},"
0274 " {0.0f, -0.39173f, 2.017f},"
0275 " {1.5958f, -0.8129f, 0.0f});"
0276 " } else {" // ITU-R 709
0277 " coefficients = float3x3("
0278 " {1.1643f, 1.1643f, 1.1643f},"
0279 " {0.0f, -0.213f, 2.112f},"
0280 " {1.793f, -0.533f, 0.0f});"
0281 " }"
0282 " out.fragColor = float4(coefficients * yuv, 1.0f);"
0283 " return out;"
0284 "}";
0285 m_fragEntryPoint = QByteArrayLiteral("main0");
0286 }
0287 }
0288
0289 FuncAndLib compileShader(const QByteArray &source, const QByteArray &entryPoint)
0290 {
0291 FuncAndLib fl;
0292
0293 NSString *srcstr = [NSString stringWithUTF8String: source.constData()];
0294 MTLCompileOptions *opts = [[MTLCompileOptions alloc] init];
0295 opts.languageVersion = MTLLanguageVersion1_2;
0296 NSError *err = nil;
0297 fl.second = [m_device newLibraryWithSource: srcstr options: opts error: &err];
0298 [opts release];
0299 // srcstr is autoreleased
0300
0301 if (err) {
0302 const QString msg = QString::fromNSString(err.localizedDescription);
0303 qFatal("%s", qPrintable(msg));
0304 return fl;
0305 }
0306
0307 NSString *name = [NSString stringWithUTF8String: entryPoint.constData()];
0308 fl.first = [fl.second newFunctionWithName: name];
0309 // [name release];
0310
0311 return fl;
0312 }
0313 };
0314 // clang-format on
0315
0316 MetalVideoWidget::MetalVideoWidget(int id, QObject *parent)
0317 : VideoWidget{id, parent}
0318 , m_renderer{new MetalVideoRenderer}
0319 {
0320 m_maxTextureSize = 16384;
0321 }
0322
0323 MetalVideoWidget::~MetalVideoWidget() {}
0324
0325 void MetalVideoWidget::initialize()
0326 {
0327 m_renderer->displayRulerHeight = qRound(m_displayRulerHeight * devicePixelRatioF() * 0.5);
0328 m_renderer->initialize(quickWindow());
0329 VideoWidget::initialize();
0330 }
0331
0332 void MetalVideoWidget::renderVideo()
0333 {
0334 m_mutex.lock();
0335 if (m_sharedFrame.is_valid()) {
0336 m_renderer->render(size(), rect(), devicePixelRatio(), zoom(), offset(), m_sharedFrame);
0337 }
0338 m_mutex.unlock();
0339 VideoWidget::renderVideo();
0340 }
0341
0342 void MetalVideoWidget::updateRulerHeight(int addedHeight)
0343 {
0344 VideoWidget::updateRulerHeight(addedHeight);
0345 m_renderer->displayRulerHeight = qRound(m_displayRulerHeight * devicePixelRatioF() * 0.5);
0346 }
0347
0348 #include "metalvideowidget.moc"