File indexing completed on 2024-05-05 04:53:39
0001 /* 0002 SPDX-FileCopyrightText: 2023 Meltytech, LLC 0003 SPDX-License-Identifier: GPL-3.0-or-later 0004 */ 0005 0006 #include "openglvideowidget.h" 0007 #include "core.h" 0008 #include "profiles/profilemodel.hpp" 0009 0010 #include <QOpenGLFunctions_1_1> 0011 #include <QOpenGLFunctions_3_2_Core> 0012 #include <QOpenGLVersionFunctionsFactory> 0013 #include <utility> 0014 0015 #ifdef QT_NO_DEBUG 0016 #define check_error(fn) \ 0017 { \ 0018 } 0019 #else 0020 #define check_error(fn) \ 0021 { \ 0022 int err = fn->glGetError(); \ 0023 if (err != GL_NO_ERROR) { \ 0024 qDebug() << "GL error" << Qt::hex << err << Qt::dec << "at" << __FILE__ << ":" << __LINE__; \ 0025 } \ 0026 } 0027 #endif 0028 0029 OpenGLVideoWidget::OpenGLVideoWidget(int id, QObject *parent) 0030 : VideoWidget{id, parent} 0031 , m_quickContext(nullptr) 0032 , m_isThreadedOpenGL(false) 0033 { 0034 m_renderTexture[0] = m_renderTexture[1] = m_renderTexture[2] = 0; 0035 m_displayTexture[0] = m_displayTexture[1] = m_displayTexture[2] = 0; 0036 } 0037 0038 OpenGLVideoWidget::~OpenGLVideoWidget() 0039 { 0040 qDebug() << "begin"; 0041 if (m_renderTexture[0] && m_displayTexture[0] && m_context) { 0042 m_context->makeCurrent(&m_offscreenSurface); 0043 m_context->functions()->glDeleteTextures(3, m_renderTexture); 0044 if (m_displayTexture[0] && m_displayTexture[1] && m_displayTexture[2]) m_context->functions()->glDeleteTextures(3, m_displayTexture); 0045 m_context->doneCurrent(); 0046 } 0047 } 0048 0049 void OpenGLVideoWidget::initialize() 0050 { 0051 qDebug() << "begin"; 0052 auto context = static_cast<QOpenGLContext *>(quickWindow()->rendererInterface()->getResource(quickWindow(), QSGRendererInterface::OpenGLContextResource)); 0053 m_quickContext = context; 0054 0055 if (!m_offscreenSurface.isValid()) { 0056 m_offscreenSurface.setFormat(context->format()); 0057 m_offscreenSurface.create(); 0058 } 0059 Q_ASSERT(m_offscreenSurface.isValid()); 0060 0061 initializeOpenGLFunctions(); 0062 qDebug() << "OpenGL vendor" << QString::fromUtf8((const char *)glGetString(GL_VENDOR)); 0063 qDebug() << "OpenGL renderer" << QString::fromUtf8((const char *)glGetString(GL_RENDERER)); 0064 qDebug() << "OpenGL threaded?" << context->supportsThreadedOpenGL(); 0065 qDebug() << "OpenGL ES?" << context->isOpenGLES(); 0066 glGetIntegerv(GL_MAX_TEXTURE_SIZE, &m_maxTextureSize); 0067 qDebug() << "OpenGL maximum texture size =" << m_maxTextureSize; 0068 GLint dims[2]; 0069 glGetIntegerv(GL_MAX_VIEWPORT_DIMS, &dims[0]); 0070 qDebug() << "OpenGL maximum viewport size =" << dims[0] << "x" << dims[1]; 0071 0072 createShader(); 0073 0074 qDebug() << "end"; 0075 VideoWidget::initialize(); 0076 } 0077 0078 void OpenGLVideoWidget::createShader() 0079 { 0080 m_shader.reset(new QOpenGLShaderProgram); 0081 m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, "uniform highp mat4 projection;" 0082 "uniform highp mat4 modelView;" 0083 "attribute highp vec4 vertex;" 0084 "attribute highp vec2 texCoord;" 0085 "varying highp vec2 coordinates;" 0086 "void main(void) {" 0087 " gl_Position = projection * modelView * vertex;" 0088 " coordinates = texCoord;" 0089 "}"); 0090 m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, 0091 "uniform sampler2D Ytex, Utex, Vtex;" 0092 "uniform lowp int colorspace;" 0093 "varying highp vec2 coordinates;" 0094 "void main(void) {" 0095 " mediump vec3 texel;" 0096 " texel.r = texture2D(Ytex, coordinates).r - 16.0/255.0;" // Y 0097 " texel.g = texture2D(Utex, coordinates).r - 128.0/255.0;" // U 0098 " texel.b = texture2D(Vtex, coordinates).r - 128.0/255.0;" // V 0099 " mediump mat3 coefficients;" 0100 " if (colorspace == 601) {" 0101 " coefficients = mat3(" 0102 " 1.1643, 1.1643, 1.1643," // column 1 0103 " 0.0, -0.39173, 2.017," // column 2 0104 " 1.5958, -0.8129, 0.0);" // column 3 0105 " } else {" // ITU-R 709 0106 " coefficients = mat3(" 0107 " 1.1643, 1.1643, 1.1643," // column 1 0108 " 0.0, -0.213, 2.112," // column 2 0109 " 1.793, -0.533, 0.0);" // column 3 0110 " }" 0111 " gl_FragColor = vec4(coefficients * texel, 1.0);" 0112 "}"); 0113 m_shader->link(); 0114 m_textureLocation[0] = m_shader->uniformLocation("Ytex"); 0115 m_textureLocation[1] = m_shader->uniformLocation("Utex"); 0116 m_textureLocation[2] = m_shader->uniformLocation("Vtex"); 0117 m_colorspaceLocation = m_shader->uniformLocation("colorspace"); 0118 m_projectionLocation = m_shader->uniformLocation("projection"); 0119 m_modelViewLocation = m_shader->uniformLocation("modelView"); 0120 m_vertexLocation = m_shader->attributeLocation("vertex"); 0121 m_texCoordLocation = m_shader->attributeLocation("texCoord"); 0122 } 0123 0124 static void uploadTextures(QOpenGLContext *context, const SharedFrame &frame, GLuint texture[]) 0125 { 0126 int width = frame.get_image_width(); 0127 int height = frame.get_image_height(); 0128 const uint8_t *image = frame.get_image(mlt_image_yuv420p); 0129 QOpenGLFunctions *f = context->functions(); 0130 0131 // The planes of pixel data may not be a multiple of the default 4 bytes. 0132 f->glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 0133 0134 // Upload each plane of YUV to a texture. 0135 if (texture[0]) f->glDeleteTextures(3, texture); 0136 check_error(f); 0137 f->glGenTextures(3, texture); 0138 check_error(f); 0139 0140 f->glBindTexture(GL_TEXTURE_2D, texture[0]); 0141 check_error(f); 0142 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 0143 check_error(f); 0144 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 0145 check_error(f); 0146 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 0147 check_error(f); 0148 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 0149 check_error(f); 0150 f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, image); 0151 check_error(f); 0152 0153 f->glBindTexture(GL_TEXTURE_2D, texture[1]); 0154 check_error(f); 0155 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 0156 check_error(f); 0157 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 0158 check_error(f); 0159 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 0160 check_error(f); 0161 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 0162 check_error(f); 0163 f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width / 2, height / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, image + width * height); 0164 check_error(f); 0165 0166 f->glBindTexture(GL_TEXTURE_2D, texture[2]); 0167 check_error(f); 0168 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 0169 check_error(f); 0170 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 0171 check_error(f); 0172 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 0173 check_error(f); 0174 f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 0175 check_error(f); 0176 f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width / 2, height / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, image + width * height + width / 2 * height / 2); 0177 check_error(f); 0178 } 0179 0180 void OpenGLVideoWidget::renderVideo() 0181 { 0182 auto context = static_cast<QOpenGLContext *>(quickWindow()->rendererInterface()->getResource(quickWindow(), QSGRendererInterface::OpenGLContextResource)); 0183 if (!m_quickContext) { 0184 qDebug() << "No quickContext"; 0185 return; 0186 } 0187 if (!context->isValid()) { 0188 qDebug() << "No QSGRendererInterface::OpenGLContextResource"; 0189 return; 0190 } 0191 #ifndef QT_NO_DEBUG 0192 QOpenGLFunctions *f = context->functions(); 0193 #endif 0194 float width = this->width() * devicePixelRatioF(); 0195 float height = this->height() * devicePixelRatioF(); 0196 0197 glDisable(GL_BLEND); 0198 glDisable(GL_DEPTH_TEST); 0199 glDepthMask(GL_FALSE); 0200 glViewport(0, qRound(m_displayRulerHeight * devicePixelRatioF() * 0.5), int(width), int(height)); 0201 check_error(f); 0202 0203 if (!m_isThreadedOpenGL) { 0204 m_mutex.lock(); 0205 if (!m_sharedFrame.is_valid()) { 0206 m_mutex.unlock(); 0207 return; 0208 } 0209 uploadTextures(context, m_sharedFrame, m_displayTexture); 0210 m_mutex.unlock(); 0211 } 0212 0213 if (!m_displayTexture[0]) { 0214 return; 0215 } 0216 quickWindow()->beginExternalCommands(); 0217 // Bind textures. 0218 for (int i = 0; i < 3; ++i) { 0219 if (m_displayTexture[i]) { 0220 glActiveTexture(GL_TEXTURE0 + i); 0221 glBindTexture(GL_TEXTURE_2D, m_displayTexture[i]); 0222 check_error(f); 0223 } 0224 } 0225 0226 // Init shader program. 0227 m_shader->bind(); 0228 m_shader->setUniformValue(m_textureLocation[0], 0); 0229 m_shader->setUniformValue(m_textureLocation[1], 1); 0230 m_shader->setUniformValue(m_textureLocation[2], 2); 0231 m_shader->setUniformValue(m_colorspaceLocation, pCore->getCurrentProfile()->colorspace()); 0232 check_error(f); 0233 0234 // Setup an orthographic projection. 0235 QMatrix4x4 projection; 0236 projection.scale(2.0f / width, 2.0f / height); 0237 m_shader->setUniformValue(m_projectionLocation, projection); 0238 check_error(f); 0239 0240 // Set model view. 0241 QMatrix4x4 modelView; 0242 if (rect().width() > 0.0 && zoom() > 0.0) { 0243 if (offset().x() || offset().y()) modelView.translate(-offset().x() * devicePixelRatioF(), offset().y() * devicePixelRatioF()); 0244 modelView.scale(zoom(), zoom()); 0245 } 0246 m_shader->setUniformValue(m_modelViewLocation, modelView); 0247 check_error(f); 0248 0249 // Provide vertices of triangle strip. 0250 QVector<QVector2D> vertices; 0251 width = rect().width() * devicePixelRatioF(); 0252 height = rect().height() * devicePixelRatioF(); 0253 vertices << QVector2D(-width / 2.0f, -height / 2.0f); 0254 vertices << QVector2D(-width / 2.0f, height / 2.0f); 0255 vertices << QVector2D(width / 2.0f, -height / 2.0f); 0256 vertices << QVector2D(width / 2.0f, height / 2.0f); 0257 m_shader->enableAttributeArray(m_vertexLocation); 0258 check_error(f); 0259 m_shader->setAttributeArray(m_vertexLocation, vertices.constData()); 0260 check_error(f); 0261 0262 // Provide texture coordinates. 0263 QVector<QVector2D> texCoord; 0264 texCoord << QVector2D(0.0f, 1.0f); 0265 texCoord << QVector2D(0.0f, 0.0f); 0266 texCoord << QVector2D(1.0f, 1.0f); 0267 texCoord << QVector2D(1.0f, 0.0f); 0268 m_shader->enableAttributeArray(m_texCoordLocation); 0269 check_error(f); 0270 m_shader->setAttributeArray(m_texCoordLocation, texCoord.constData()); 0271 check_error(f); 0272 0273 // Render 0274 glDrawArrays(GL_TRIANGLE_STRIP, 0, vertices.size()); 0275 check_error(f); 0276 0277 if (m_sendFrame && m_analyseSem.tryAcquire(1)) { 0278 // Render RGB frame for analysis 0279 if (!qFuzzyCompare(m_zoom, 1.0f)) { 0280 // Disable monitor zoom to render frame 0281 modelView = QMatrix4x4(); 0282 m_shader->setUniformValue(m_modelViewLocation, modelView); 0283 } 0284 if ((m_fbo == nullptr) || m_fbo->size() != m_profileSize) { 0285 delete m_fbo; 0286 QOpenGLFramebufferObjectFormat fmt; 0287 fmt.setSamples(1); 0288 m_fbo = new QOpenGLFramebufferObject(m_profileSize.width(), m_profileSize.height(), fmt); // GL_TEXTURE_2D); 0289 } 0290 m_fbo->bind(); 0291 glViewport(0, 0, m_profileSize.width(), m_profileSize.height()); 0292 0293 QMatrix4x4 projection2; 0294 projection2.scale(2.0f / width, 2.0f / height); 0295 m_shader->setUniformValue(m_projectionLocation, projection2); 0296 0297 glDrawArrays(GL_TRIANGLE_STRIP, 0, vertices.size()); 0298 check_error(f); 0299 m_fbo->release(); 0300 Q_EMIT analyseFrame(m_fbo->toImage()); 0301 m_sendFrame = false; 0302 } 0303 0304 // Cleanup 0305 m_shader->disableAttributeArray(m_vertexLocation); 0306 m_shader->disableAttributeArray(m_texCoordLocation); 0307 m_shader->release(); 0308 for (int i = 0; i < 3; ++i) { 0309 if (m_displayTexture[i]) { 0310 glActiveTexture(GL_TEXTURE0 + i); 0311 glBindTexture(GL_TEXTURE_2D, 0); 0312 check_error(f); 0313 } 0314 } 0315 glActiveTexture(GL_TEXTURE0); 0316 check_error(f); 0317 0318 quickWindow()->endExternalCommands(); 0319 VideoWidget::renderVideo(); 0320 } 0321 0322 void OpenGLVideoWidget::onFrameDisplayed(const SharedFrame &frame) 0323 { 0324 if (m_isThreadedOpenGL && !m_context) { 0325 m_context.reset(new QOpenGLContext); 0326 if (m_context) { 0327 m_context->setFormat(m_quickContext->format()); 0328 m_context->setShareContext(m_quickContext); 0329 m_context->create(); 0330 } 0331 } 0332 if (m_context && m_context->isValid()) { 0333 // Using threaded OpenGL to upload textures. 0334 QOpenGLFunctions *f = m_context->functions(); 0335 m_context->makeCurrent(&m_offscreenSurface); 0336 uploadTextures(m_context.get(), frame, m_renderTexture); 0337 f->glBindTexture(GL_TEXTURE_2D, 0); 0338 check_error(f); 0339 f->glFinish(); 0340 m_context->doneCurrent(); 0341 0342 m_mutex.lock(); 0343 for (int i = 0; i < 3; ++i) 0344 std::swap(m_renderTexture[i], m_displayTexture[i]); 0345 m_mutex.unlock(); 0346 } 0347 VideoWidget::onFrameDisplayed(frame); 0348 } 0349 0350 const QStringList OpenGLVideoWidget::getGPUInfo() 0351 { 0352 if (!m_isInitialized) { 0353 return {}; 0354 } 0355 return {QString::fromUtf8((const char *)glGetString(GL_VENDOR)), QString::fromUtf8((const char *)glGetString(GL_RENDERER))}; 0356 }