File indexing completed on 2024-03-24 17:02:23
0001 /* 0002 Render a PipeWire stream into a QtQuick scene as a standard Item 0003 SPDX-FileCopyrightText: 2020 Aleix Pol Gonzalez <aleixpol@kde.org> 0004 0005 SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL 0006 */ 0007 0008 #include "pipewiresourceitem.h" 0009 #include "glhelpers.h" 0010 #include "logging.h" 0011 #include "pipewiresourcestream.h" 0012 0013 #include <QGuiApplication> 0014 #include <QOpenGLContext> 0015 #include <QOpenGLTexture> 0016 #include <QPainter> 0017 #include <QQuickWindow> 0018 #include <QRunnable> 0019 #include <QSGImageNode> 0020 #include <QSocketNotifier> 0021 #include <QThread> 0022 #include <qpa/qplatformnativeinterface.h> 0023 0024 #include <EGL/eglext.h> 0025 #include <fcntl.h> 0026 #include <libdrm/drm_fourcc.h> 0027 #include <unistd.h> 0028 0029 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0030 #include <QtPlatformHeaders/QEGLNativeContext> 0031 #endif 0032 0033 static void pwInit() 0034 { 0035 pw_init(nullptr, nullptr); 0036 } 0037 Q_COREAPP_STARTUP_FUNCTION(pwInit); 0038 0039 class PipeWireSourceItemPrivate 0040 { 0041 public: 0042 uint m_nodeId = 0; 0043 std::optional<uint> m_fd; 0044 std::function<QSGTexture *()> m_createNextTexture; 0045 QScopedPointer<PipeWireSourceStream> m_stream; 0046 QScopedPointer<QOpenGLTexture> m_texture; 0047 0048 EGLImage m_image = nullptr; 0049 bool m_needsRecreateTexture = false; 0050 0051 struct { 0052 QImage texture; 0053 std::optional<QPoint> position; 0054 QPoint hotspot; 0055 bool dirty = false; 0056 } m_cursor; 0057 std::optional<QRegion> m_damage; 0058 }; 0059 0060 class DiscardEglPixmapRunnable : public QRunnable 0061 { 0062 public: 0063 DiscardEglPixmapRunnable(EGLImageKHR image, QOpenGLTexture *texture) 0064 : m_image(image) 0065 , m_texture(texture) 0066 { 0067 } 0068 0069 void run() override 0070 { 0071 if (m_image != EGL_NO_IMAGE_KHR) { 0072 eglDestroyImageKHR(eglGetCurrentDisplay(), m_image); 0073 } 0074 0075 delete m_texture; 0076 } 0077 0078 private: 0079 const EGLImageKHR m_image; 0080 QOpenGLTexture *m_texture; 0081 }; 0082 0083 PipeWireSourceItem::PipeWireSourceItem(QQuickItem *parent) 0084 : QQuickItem(parent) 0085 , d(new PipeWireSourceItemPrivate) 0086 { 0087 setFlag(ItemHasContents, true); 0088 0089 connect(this, &QQuickItem::visibleChanged, this, [this]() { 0090 setEnabled(isVisible()); 0091 if (d->m_stream) 0092 d->m_stream->setActive(isVisible()); 0093 }); 0094 } 0095 0096 PipeWireSourceItem::~PipeWireSourceItem() 0097 { 0098 if (d->m_fd) { 0099 close(*d->m_fd); 0100 } 0101 } 0102 0103 void PipeWireSourceItem::itemChange(QQuickItem::ItemChange change, const QQuickItem::ItemChangeData &data) 0104 { 0105 switch (change) { 0106 case ItemVisibleHasChanged: 0107 setEnabled(isVisible()); 0108 if (d->m_stream) 0109 d->m_stream->setActive(isVisible() && data.boolValue && isComponentComplete()); 0110 break; 0111 case ItemSceneChange: 0112 d->m_needsRecreateTexture = true; 0113 releaseResources(); 0114 break; 0115 default: 0116 break; 0117 } 0118 0119 QQuickItem::itemChange(change, data); 0120 } 0121 0122 void PipeWireSourceItem::releaseResources() 0123 { 0124 if (window()) { 0125 window()->scheduleRenderJob(new DiscardEglPixmapRunnable(d->m_image, d->m_texture.take()), QQuickWindow::NoStage); 0126 d->m_image = EGL_NO_IMAGE_KHR; 0127 } 0128 } 0129 0130 void PipeWireSourceItem::setFd(uint fd) 0131 { 0132 if (fd == d->m_fd) 0133 return; 0134 0135 if (d->m_fd) { 0136 close(*d->m_fd); 0137 } 0138 d->m_fd = fd; 0139 refresh(); 0140 Q_EMIT fdChanged(fd); 0141 } 0142 0143 void PipeWireSourceItem::resetFd() 0144 { 0145 if (!d->m_fd.has_value()) { 0146 return; 0147 } 0148 0149 setEnabled(false); 0150 close(*d->m_fd); 0151 d->m_fd.reset(); 0152 d->m_stream.reset(nullptr); 0153 d->m_createNextTexture = [] { 0154 return nullptr; 0155 }; 0156 Q_EMIT streamSizeChanged(); 0157 } 0158 0159 void PipeWireSourceItem::refresh() 0160 { 0161 setEnabled(false); 0162 0163 if (!isComponentComplete()) { 0164 return; 0165 } 0166 0167 if (d->m_nodeId == 0) { 0168 d->m_stream.reset(nullptr); 0169 Q_EMIT streamSizeChanged(); 0170 0171 d->m_createNextTexture = [] { 0172 return nullptr; 0173 }; 0174 } else { 0175 d->m_stream.reset(new PipeWireSourceStream(this)); 0176 Q_EMIT streamSizeChanged(); 0177 connect(d->m_stream.get(), &PipeWireSourceStream::streamParametersChanged, this, &PipeWireSourceItem::streamSizeChanged); 0178 0179 d->m_stream->createStream(d->m_nodeId, d->m_fd.value_or(0)); 0180 if (!d->m_stream->error().isEmpty()) { 0181 d->m_stream.reset(nullptr); 0182 d->m_nodeId = 0; 0183 return; 0184 } 0185 d->m_stream->setActive(isVisible() && isComponentComplete()); 0186 0187 connect(d->m_stream.data(), &PipeWireSourceStream::frameReceived, this, &PipeWireSourceItem::processFrame); 0188 connect(d->m_stream.data(), &PipeWireSourceStream::stateChanged, this, &PipeWireSourceItem::stateChanged); 0189 } 0190 Q_EMIT stateChanged(); 0191 } 0192 0193 void PipeWireSourceItem::setNodeId(uint nodeId) 0194 { 0195 if (nodeId == d->m_nodeId) 0196 return; 0197 0198 d->m_nodeId = nodeId; 0199 refresh(); 0200 Q_EMIT nodeIdChanged(nodeId); 0201 } 0202 0203 class PipeWireRenderNode : public QSGNode 0204 { 0205 public: 0206 QSGImageNode *screenNode(QQuickWindow *window) 0207 { 0208 if (!m_screenNode) { 0209 m_screenNode = window->createImageNode(); 0210 appendChildNode(m_screenNode); 0211 } 0212 return m_screenNode; 0213 } 0214 QSGImageNode *cursorNode(QQuickWindow *window) 0215 { 0216 if (!m_cursorNode) { 0217 m_cursorNode = window->createImageNode(); 0218 appendChildNode(m_cursorNode); 0219 } 0220 return m_cursorNode; 0221 } 0222 0223 QSGImageNode *damageNode(QQuickWindow *window) 0224 { 0225 if (!m_damageNode) { 0226 m_damageNode = window->createImageNode(); 0227 appendChildNode(m_damageNode); 0228 } 0229 return m_damageNode; 0230 } 0231 0232 void discardCursor() 0233 { 0234 if (m_cursorNode) { 0235 removeChildNode(m_cursorNode); 0236 delete m_cursorNode; 0237 m_cursorNode = nullptr; 0238 } 0239 } 0240 0241 void discardDamage() 0242 { 0243 if (m_damageNode) { 0244 removeChildNode(m_damageNode); 0245 delete m_damageNode; 0246 m_damageNode = nullptr; 0247 } 0248 } 0249 0250 private: 0251 QSGImageNode *m_screenNode = nullptr; 0252 QSGImageNode *m_cursorNode = nullptr; 0253 QSGImageNode *m_damageNode = nullptr; 0254 }; 0255 0256 QSGNode *PipeWireSourceItem::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNodeData *) 0257 { 0258 if (Q_UNLIKELY(!d->m_createNextTexture)) { 0259 return node; 0260 } 0261 0262 auto texture = d->m_createNextTexture(); 0263 if (!texture) { 0264 delete node; 0265 return nullptr; 0266 } 0267 0268 QSGImageNode *screenNode; 0269 auto pwNode = dynamic_cast<PipeWireRenderNode *>(node); 0270 if (!pwNode) { 0271 delete node; 0272 pwNode = new PipeWireRenderNode; 0273 screenNode = window()->createImageNode(); 0274 screenNode->setOwnsTexture(true); 0275 pwNode->appendChildNode(screenNode); 0276 } else { 0277 screenNode = static_cast<QSGImageNode *>(pwNode->childAtIndex(0)); 0278 } 0279 screenNode->setTexture(texture); 0280 0281 const auto br = boundingRect().toRect(); 0282 QRect rect({0, 0}, texture->textureSize().scaled(br.size(), Qt::KeepAspectRatio)); 0283 rect.moveCenter(br.center()); 0284 screenNode->setRect(rect); 0285 0286 if (!d->m_cursor.position.has_value() || d->m_cursor.texture.isNull()) { 0287 pwNode->discardCursor(); 0288 } else { 0289 QSGImageNode *cursorNode = pwNode->cursorNode(window()); 0290 if (d->m_cursor.dirty || !cursorNode->texture()) { 0291 cursorNode->setTexture(window()->createTextureFromImage(d->m_cursor.texture)); 0292 d->m_cursor.dirty = false; 0293 } 0294 const qreal scale = qreal(rect.width()) / texture->textureSize().width(); 0295 cursorNode->setRect(QRectF{rect.topLeft() + (d->m_cursor.position.value() * scale), d->m_cursor.texture.size() * scale}); 0296 Q_ASSERT(cursorNode->texture()); 0297 } 0298 0299 if (!d->m_damage || d->m_damage->isEmpty()) { 0300 pwNode->discardDamage(); 0301 } else { 0302 auto *damageNode = pwNode->damageNode(window()); 0303 QImage damageImage(texture->textureSize(), QImage::Format_RGBA64_Premultiplied); 0304 damageImage.fill(Qt::transparent); 0305 QPainter p(&damageImage); 0306 p.setBrush(Qt::red); 0307 for (auto rect : *d->m_damage) { 0308 p.drawRect(rect); 0309 } 0310 damageNode->setTexture(window()->createTextureFromImage(damageImage)); 0311 damageNode->setRect(rect); 0312 Q_ASSERT(damageNode->texture()); 0313 } 0314 return pwNode; 0315 } 0316 0317 QString PipeWireSourceItem::error() const 0318 { 0319 return d->m_stream->error(); 0320 } 0321 0322 void PipeWireSourceItem::processFrame(const PipeWireFrame &frame) 0323 { 0324 d->m_damage = frame.damage; 0325 0326 if (frame.cursor) { 0327 d->m_cursor.position = frame.cursor->position; 0328 d->m_cursor.hotspot = frame.cursor->hotspot; 0329 if (!frame.cursor->texture.isNull()) { 0330 d->m_cursor.dirty = true; 0331 d->m_cursor.texture = frame.cursor->texture; 0332 } 0333 } else { 0334 d->m_cursor.position = std::nullopt; 0335 d->m_cursor.hotspot = {}; 0336 } 0337 0338 if (frame.dmabuf) { 0339 updateTextureDmaBuf(*frame.dmabuf, frame.format); 0340 } else if (frame.image) { 0341 updateTextureImage(*frame.image); 0342 } 0343 0344 if (window() && window()->isVisible()) { 0345 update(); 0346 } 0347 } 0348 0349 void PipeWireSourceItem::updateTextureDmaBuf(const DmaBufAttributes &attribs, spa_video_format format) 0350 { 0351 if (!window()) { 0352 qCWarning(PIPEWIRE_LOGGING) << "Window not available" << this; 0353 return; 0354 } 0355 0356 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0357 const auto openglContext = window()->openglContext(); 0358 #else 0359 const auto openglContext = static_cast<QOpenGLContext *>(window()->rendererInterface()->getResource(window(), QSGRendererInterface::OpenGLContextResource)); 0360 #endif 0361 if (!openglContext || !d->m_stream) { 0362 qCWarning(PIPEWIRE_LOGGING) << "need a window and a context" << window(); 0363 return; 0364 } 0365 0366 d->m_createNextTexture = [this, format, attribs]() -> QSGTexture * { 0367 const EGLDisplay display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay")); 0368 if (d->m_image) { 0369 eglDestroyImageKHR(display, d->m_image); 0370 } 0371 const auto size = d->m_stream->size(); 0372 d->m_image = GLHelpers::createImage(display, attribs, PipeWireSourceStream::spaVideoFormatToDrmFormat(format), size, nullptr); 0373 if (d->m_image == EGL_NO_IMAGE_KHR) { 0374 d->m_stream->renegotiateModifierFailed(format, attribs.modifier); 0375 return nullptr; 0376 } 0377 if (!d->m_texture) { 0378 d->m_texture.reset(new QOpenGLTexture(QOpenGLTexture::Target2D)); 0379 bool created = d->m_texture->create(); 0380 Q_ASSERT(created); 0381 } 0382 0383 GLHelpers::initDebugOutput(); 0384 d->m_texture->bind(); 0385 0386 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)d->m_image); 0387 0388 d->m_texture->setWrapMode(QOpenGLTexture::ClampToEdge); 0389 d->m_texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); 0390 d->m_texture->release(); 0391 d->m_texture->setSize(size.width(), size.height()); 0392 0393 int textureId = d->m_texture->textureId(); 0394 QQuickWindow::CreateTextureOption textureOption = 0395 format == SPA_VIDEO_FORMAT_ARGB || format == SPA_VIDEO_FORMAT_BGRA ? QQuickWindow::TextureHasAlphaChannel : QQuickWindow::TextureIsOpaque; 0396 setEnabled(true); 0397 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0398 return window()->createTextureFromNativeObject(QQuickWindow::NativeObjectTexture, &textureId, 0 /*a vulkan thing?*/, size, textureOption); 0399 #else 0400 return QNativeInterface::QSGOpenGLTexture::fromNative(textureId, window(), size, textureOption); 0401 #endif 0402 }; 0403 } 0404 0405 void PipeWireSourceItem::updateTextureImage(const QImage &image) 0406 { 0407 if (!window()) { 0408 qCWarning(PIPEWIRE_LOGGING) << "pass"; 0409 return; 0410 } 0411 0412 d->m_createNextTexture = [this, image] { 0413 setEnabled(true); 0414 return window()->createTextureFromImage(image, QQuickWindow::TextureIsOpaque); 0415 }; 0416 } 0417 0418 void PipeWireSourceItem::componentComplete() 0419 { 0420 QQuickItem::componentComplete(); 0421 if (d->m_nodeId != 0) { 0422 refresh(); 0423 } 0424 } 0425 0426 PipeWireSourceItem::StreamState PipeWireSourceItem::state() const 0427 { 0428 if (!d->m_stream) { 0429 return StreamState::Unconnected; 0430 } 0431 switch (d->m_stream->state()) { 0432 case PW_STREAM_STATE_ERROR: 0433 return StreamState::Error; 0434 case PW_STREAM_STATE_UNCONNECTED: 0435 return StreamState::Unconnected; 0436 case PW_STREAM_STATE_CONNECTING: 0437 return StreamState::Connecting; 0438 case PW_STREAM_STATE_PAUSED: 0439 return StreamState::Paused; 0440 case PW_STREAM_STATE_STREAMING: 0441 return StreamState::Streaming; 0442 default: 0443 return StreamState::Error; 0444 } 0445 } 0446 0447 uint PipeWireSourceItem::fd() const 0448 { 0449 return d->m_fd.value_or(0); 0450 } 0451 0452 uint PipeWireSourceItem::nodeId() const 0453 { 0454 return d->m_nodeId; 0455 } 0456 0457 QSize PipeWireSourceItem::streamSize() const 0458 { 0459 if (!d->m_stream) { 0460 return QSize(); 0461 } 0462 return d->m_stream->size(); 0463 }