File indexing completed on 2024-03-24 17:02:23

0001 /*
0002     Render a PipeWire stream into a QtQuick scene as a standard Item
0003     SPDX-FileCopyrightText: 2020 Aleix Pol Gonzalez <aleixpol@kde.org>
0004 
0005     SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
0006 */
0007 
0008 #include "pipewiresourceitem.h"
0009 #include "glhelpers.h"
0010 #include "logging.h"
0011 #include "pipewiresourcestream.h"
0012 
0013 #include <QGuiApplication>
0014 #include <QOpenGLContext>
0015 #include <QOpenGLTexture>
0016 #include <QPainter>
0017 #include <QQuickWindow>
0018 #include <QRunnable>
0019 #include <QSGImageNode>
0020 #include <QSocketNotifier>
0021 #include <QThread>
0022 #include <qpa/qplatformnativeinterface.h>
0023 
0024 #include <EGL/eglext.h>
0025 #include <fcntl.h>
0026 #include <libdrm/drm_fourcc.h>
0027 #include <unistd.h>
0028 
0029 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
0030 #include <QtPlatformHeaders/QEGLNativeContext>
0031 #endif
0032 
0033 static void pwInit()
0034 {
0035     pw_init(nullptr, nullptr);
0036 }
0037 Q_COREAPP_STARTUP_FUNCTION(pwInit);
0038 
0039 class PipeWireSourceItemPrivate
0040 {
0041 public:
0042     uint m_nodeId = 0;
0043     std::optional<uint> m_fd;
0044     std::function<QSGTexture *()> m_createNextTexture;
0045     QScopedPointer<PipeWireSourceStream> m_stream;
0046     QScopedPointer<QOpenGLTexture> m_texture;
0047 
0048     EGLImage m_image = nullptr;
0049     bool m_needsRecreateTexture = false;
0050 
0051     struct {
0052         QImage texture;
0053         std::optional<QPoint> position;
0054         QPoint hotspot;
0055         bool dirty = false;
0056     } m_cursor;
0057     std::optional<QRegion> m_damage;
0058 };
0059 
0060 class DiscardEglPixmapRunnable : public QRunnable
0061 {
0062 public:
0063     DiscardEglPixmapRunnable(EGLImageKHR image, QOpenGLTexture *texture)
0064         : m_image(image)
0065         , m_texture(texture)
0066     {
0067     }
0068 
0069     void run() override
0070     {
0071         if (m_image != EGL_NO_IMAGE_KHR) {
0072             eglDestroyImageKHR(eglGetCurrentDisplay(), m_image);
0073         }
0074 
0075         delete m_texture;
0076     }
0077 
0078 private:
0079     const EGLImageKHR m_image;
0080     QOpenGLTexture *m_texture;
0081 };
0082 
0083 PipeWireSourceItem::PipeWireSourceItem(QQuickItem *parent)
0084     : QQuickItem(parent)
0085     , d(new PipeWireSourceItemPrivate)
0086 {
0087     setFlag(ItemHasContents, true);
0088 
0089     connect(this, &QQuickItem::visibleChanged, this, [this]() {
0090         setEnabled(isVisible());
0091         if (d->m_stream)
0092             d->m_stream->setActive(isVisible());
0093     });
0094 }
0095 
0096 PipeWireSourceItem::~PipeWireSourceItem()
0097 {
0098     if (d->m_fd) {
0099         close(*d->m_fd);
0100     }
0101 }
0102 
0103 void PipeWireSourceItem::itemChange(QQuickItem::ItemChange change, const QQuickItem::ItemChangeData &data)
0104 {
0105     switch (change) {
0106     case ItemVisibleHasChanged:
0107         setEnabled(isVisible());
0108         if (d->m_stream)
0109             d->m_stream->setActive(isVisible() && data.boolValue && isComponentComplete());
0110         break;
0111     case ItemSceneChange:
0112         d->m_needsRecreateTexture = true;
0113         releaseResources();
0114         break;
0115     default:
0116         break;
0117     }
0118 
0119     QQuickItem::itemChange(change, data);
0120 }
0121 
0122 void PipeWireSourceItem::releaseResources()
0123 {
0124     if (window()) {
0125         window()->scheduleRenderJob(new DiscardEglPixmapRunnable(d->m_image, d->m_texture.take()), QQuickWindow::NoStage);
0126         d->m_image = EGL_NO_IMAGE_KHR;
0127     }
0128 }
0129 
0130 void PipeWireSourceItem::setFd(uint fd)
0131 {
0132     if (fd == d->m_fd)
0133         return;
0134 
0135     if (d->m_fd) {
0136         close(*d->m_fd);
0137     }
0138     d->m_fd = fd;
0139     refresh();
0140     Q_EMIT fdChanged(fd);
0141 }
0142 
0143 void PipeWireSourceItem::resetFd()
0144 {
0145     if (!d->m_fd.has_value()) {
0146         return;
0147     }
0148 
0149     setEnabled(false);
0150     close(*d->m_fd);
0151     d->m_fd.reset();
0152     d->m_stream.reset(nullptr);
0153     d->m_createNextTexture = [] {
0154         return nullptr;
0155     };
0156     Q_EMIT streamSizeChanged();
0157 }
0158 
0159 void PipeWireSourceItem::refresh()
0160 {
0161     setEnabled(false);
0162 
0163     if (!isComponentComplete()) {
0164         return;
0165     }
0166 
0167     if (d->m_nodeId == 0) {
0168         d->m_stream.reset(nullptr);
0169         Q_EMIT streamSizeChanged();
0170 
0171         d->m_createNextTexture = [] {
0172             return nullptr;
0173         };
0174     } else {
0175         d->m_stream.reset(new PipeWireSourceStream(this));
0176         Q_EMIT streamSizeChanged();
0177         connect(d->m_stream.get(), &PipeWireSourceStream::streamParametersChanged, this, &PipeWireSourceItem::streamSizeChanged);
0178 
0179         d->m_stream->createStream(d->m_nodeId, d->m_fd.value_or(0));
0180         if (!d->m_stream->error().isEmpty()) {
0181             d->m_stream.reset(nullptr);
0182             d->m_nodeId = 0;
0183             return;
0184         }
0185         d->m_stream->setActive(isVisible() && isComponentComplete());
0186 
0187         connect(d->m_stream.data(), &PipeWireSourceStream::frameReceived, this, &PipeWireSourceItem::processFrame);
0188         connect(d->m_stream.data(), &PipeWireSourceStream::stateChanged, this, &PipeWireSourceItem::stateChanged);
0189     }
0190     Q_EMIT stateChanged();
0191 }
0192 
0193 void PipeWireSourceItem::setNodeId(uint nodeId)
0194 {
0195     if (nodeId == d->m_nodeId)
0196         return;
0197 
0198     d->m_nodeId = nodeId;
0199     refresh();
0200     Q_EMIT nodeIdChanged(nodeId);
0201 }
0202 
0203 class PipeWireRenderNode : public QSGNode
0204 {
0205 public:
0206     QSGImageNode *screenNode(QQuickWindow *window)
0207     {
0208         if (!m_screenNode) {
0209             m_screenNode = window->createImageNode();
0210             appendChildNode(m_screenNode);
0211         }
0212         return m_screenNode;
0213     }
0214     QSGImageNode *cursorNode(QQuickWindow *window)
0215     {
0216         if (!m_cursorNode) {
0217             m_cursorNode = window->createImageNode();
0218             appendChildNode(m_cursorNode);
0219         }
0220         return m_cursorNode;
0221     }
0222 
0223     QSGImageNode *damageNode(QQuickWindow *window)
0224     {
0225         if (!m_damageNode) {
0226             m_damageNode = window->createImageNode();
0227             appendChildNode(m_damageNode);
0228         }
0229         return m_damageNode;
0230     }
0231 
0232     void discardCursor()
0233     {
0234         if (m_cursorNode) {
0235             removeChildNode(m_cursorNode);
0236             delete m_cursorNode;
0237             m_cursorNode = nullptr;
0238         }
0239     }
0240 
0241     void discardDamage()
0242     {
0243         if (m_damageNode) {
0244             removeChildNode(m_damageNode);
0245             delete m_damageNode;
0246             m_damageNode = nullptr;
0247         }
0248     }
0249 
0250 private:
0251     QSGImageNode *m_screenNode = nullptr;
0252     QSGImageNode *m_cursorNode = nullptr;
0253     QSGImageNode *m_damageNode = nullptr;
0254 };
0255 
0256 QSGNode *PipeWireSourceItem::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNodeData *)
0257 {
0258     if (Q_UNLIKELY(!d->m_createNextTexture)) {
0259         return node;
0260     }
0261 
0262     auto texture = d->m_createNextTexture();
0263     if (!texture) {
0264         delete node;
0265         return nullptr;
0266     }
0267 
0268     QSGImageNode *screenNode;
0269     auto pwNode = dynamic_cast<PipeWireRenderNode *>(node);
0270     if (!pwNode) {
0271         delete node;
0272         pwNode = new PipeWireRenderNode;
0273         screenNode = window()->createImageNode();
0274         screenNode->setOwnsTexture(true);
0275         pwNode->appendChildNode(screenNode);
0276     } else {
0277         screenNode = static_cast<QSGImageNode *>(pwNode->childAtIndex(0));
0278     }
0279     screenNode->setTexture(texture);
0280 
0281     const auto br = boundingRect().toRect();
0282     QRect rect({0, 0}, texture->textureSize().scaled(br.size(), Qt::KeepAspectRatio));
0283     rect.moveCenter(br.center());
0284     screenNode->setRect(rect);
0285 
0286     if (!d->m_cursor.position.has_value() || d->m_cursor.texture.isNull()) {
0287         pwNode->discardCursor();
0288     } else {
0289         QSGImageNode *cursorNode = pwNode->cursorNode(window());
0290         if (d->m_cursor.dirty || !cursorNode->texture()) {
0291             cursorNode->setTexture(window()->createTextureFromImage(d->m_cursor.texture));
0292             d->m_cursor.dirty = false;
0293         }
0294         const qreal scale = qreal(rect.width()) / texture->textureSize().width();
0295         cursorNode->setRect(QRectF{rect.topLeft() + (d->m_cursor.position.value() * scale), d->m_cursor.texture.size() * scale});
0296         Q_ASSERT(cursorNode->texture());
0297     }
0298 
0299     if (!d->m_damage || d->m_damage->isEmpty()) {
0300         pwNode->discardDamage();
0301     } else {
0302         auto *damageNode = pwNode->damageNode(window());
0303         QImage damageImage(texture->textureSize(), QImage::Format_RGBA64_Premultiplied);
0304         damageImage.fill(Qt::transparent);
0305         QPainter p(&damageImage);
0306         p.setBrush(Qt::red);
0307         for (auto rect : *d->m_damage) {
0308             p.drawRect(rect);
0309         }
0310         damageNode->setTexture(window()->createTextureFromImage(damageImage));
0311         damageNode->setRect(rect);
0312         Q_ASSERT(damageNode->texture());
0313     }
0314     return pwNode;
0315 }
0316 
0317 QString PipeWireSourceItem::error() const
0318 {
0319     return d->m_stream->error();
0320 }
0321 
0322 void PipeWireSourceItem::processFrame(const PipeWireFrame &frame)
0323 {
0324     d->m_damage = frame.damage;
0325 
0326     if (frame.cursor) {
0327         d->m_cursor.position = frame.cursor->position;
0328         d->m_cursor.hotspot = frame.cursor->hotspot;
0329         if (!frame.cursor->texture.isNull()) {
0330             d->m_cursor.dirty = true;
0331             d->m_cursor.texture = frame.cursor->texture;
0332         }
0333     } else {
0334         d->m_cursor.position = std::nullopt;
0335         d->m_cursor.hotspot = {};
0336     }
0337 
0338     if (frame.dmabuf) {
0339         updateTextureDmaBuf(*frame.dmabuf, frame.format);
0340     } else if (frame.image) {
0341         updateTextureImage(*frame.image);
0342     }
0343 
0344     if (window() && window()->isVisible()) {
0345         update();
0346     }
0347 }
0348 
0349 void PipeWireSourceItem::updateTextureDmaBuf(const DmaBufAttributes &attribs, spa_video_format format)
0350 {
0351     if (!window()) {
0352         qCWarning(PIPEWIRE_LOGGING) << "Window not available" << this;
0353         return;
0354     }
0355 
0356 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
0357     const auto openglContext = window()->openglContext();
0358 #else
0359     const auto openglContext = static_cast<QOpenGLContext *>(window()->rendererInterface()->getResource(window(), QSGRendererInterface::OpenGLContextResource));
0360 #endif
0361     if (!openglContext || !d->m_stream) {
0362         qCWarning(PIPEWIRE_LOGGING) << "need a window and a context" << window();
0363         return;
0364     }
0365 
0366     d->m_createNextTexture = [this, format, attribs]() -> QSGTexture * {
0367         const EGLDisplay display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay"));
0368         if (d->m_image) {
0369             eglDestroyImageKHR(display, d->m_image);
0370         }
0371         const auto size = d->m_stream->size();
0372         d->m_image = GLHelpers::createImage(display, attribs, PipeWireSourceStream::spaVideoFormatToDrmFormat(format), size, nullptr);
0373         if (d->m_image == EGL_NO_IMAGE_KHR) {
0374             d->m_stream->renegotiateModifierFailed(format, attribs.modifier);
0375             return nullptr;
0376         }
0377         if (!d->m_texture) {
0378             d->m_texture.reset(new QOpenGLTexture(QOpenGLTexture::Target2D));
0379             bool created = d->m_texture->create();
0380             Q_ASSERT(created);
0381         }
0382 
0383         GLHelpers::initDebugOutput();
0384         d->m_texture->bind();
0385 
0386         glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)d->m_image);
0387 
0388         d->m_texture->setWrapMode(QOpenGLTexture::ClampToEdge);
0389         d->m_texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear);
0390         d->m_texture->release();
0391         d->m_texture->setSize(size.width(), size.height());
0392 
0393         int textureId = d->m_texture->textureId();
0394         QQuickWindow::CreateTextureOption textureOption =
0395             format == SPA_VIDEO_FORMAT_ARGB || format == SPA_VIDEO_FORMAT_BGRA ? QQuickWindow::TextureHasAlphaChannel : QQuickWindow::TextureIsOpaque;
0396         setEnabled(true);
0397 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
0398         return window()->createTextureFromNativeObject(QQuickWindow::NativeObjectTexture, &textureId, 0 /*a vulkan thing?*/, size, textureOption);
0399 #else
0400         return QNativeInterface::QSGOpenGLTexture::fromNative(textureId, window(), size, textureOption);
0401 #endif
0402     };
0403 }
0404 
0405 void PipeWireSourceItem::updateTextureImage(const QImage &image)
0406 {
0407     if (!window()) {
0408         qCWarning(PIPEWIRE_LOGGING) << "pass";
0409         return;
0410     }
0411 
0412     d->m_createNextTexture = [this, image] {
0413         setEnabled(true);
0414         return window()->createTextureFromImage(image, QQuickWindow::TextureIsOpaque);
0415     };
0416 }
0417 
0418 void PipeWireSourceItem::componentComplete()
0419 {
0420     QQuickItem::componentComplete();
0421     if (d->m_nodeId != 0) {
0422         refresh();
0423     }
0424 }
0425 
0426 PipeWireSourceItem::StreamState PipeWireSourceItem::state() const
0427 {
0428     if (!d->m_stream) {
0429         return StreamState::Unconnected;
0430     }
0431     switch (d->m_stream->state()) {
0432     case PW_STREAM_STATE_ERROR:
0433         return StreamState::Error;
0434     case PW_STREAM_STATE_UNCONNECTED:
0435         return StreamState::Unconnected;
0436     case PW_STREAM_STATE_CONNECTING:
0437         return StreamState::Connecting;
0438     case PW_STREAM_STATE_PAUSED:
0439         return StreamState::Paused;
0440     case PW_STREAM_STATE_STREAMING:
0441         return StreamState::Streaming;
0442     default:
0443         return StreamState::Error;
0444     }
0445 }
0446 
0447 uint PipeWireSourceItem::fd() const
0448 {
0449     return d->m_fd.value_or(0);
0450 }
0451 
0452 uint PipeWireSourceItem::nodeId() const
0453 {
0454     return d->m_nodeId;
0455 }
0456 
0457 QSize PipeWireSourceItem::streamSize() const
0458 {
0459     if (!d->m_stream) {
0460         return QSize();
0461     }
0462     return d->m_stream->size();
0463 }