kpipewire/src/pipewiresourceitem.cpp

0001 /*
0002     Render a PipeWire stream into a QtQuick scene as a standard Item
0003     SPDX-FileCopyrightText: 2020 Aleix Pol Gonzalez <aleixpol@kde.org>
0004
0005     SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
0006 */
0007
0008 #include "pipewiresourceitem.h"
0009 #include "glhelpers.h"
0010 #include "logging.h"
0011 #include "pipewiresourcestream.h"
0012 #include "pwhelpers.h"
0013
0014 #include <QGuiApplication>
0015 #include <QOpenGLContext>
0016 #include <QOpenGLTexture>
0017 #include <QPainter>
0018 #include <QQuickWindow>
0019 #include <QRunnable>
0020 #include <QSGImageNode>
0021 #include <QSocketNotifier>
0022 #include <QThread>
0023 #include <qpa/qplatformnativeinterface.h>
0024
0025 #include <EGL/eglext.h>
0026 #include <fcntl.h>
0027 #include <libdrm/drm_fourcc.h>
0028 #include <unistd.h>
0029
0030 static void pwInit()
0031 {
0032     pw_init(nullptr, nullptr);
0033 }
0034 Q_COREAPP_STARTUP_FUNCTION(pwInit);
0035
0036 class PipeWireSourceItemPrivate
0037 {
0038 public:
0039     uint m_nodeId = 0;
0040     std::optional<uint> m_fd;
0041     std::function<QSGTexture *()> m_createNextTexture;
0042     std::unique_ptr<PipeWireSourceStream> m_stream;
0043     std::unique_ptr<QOpenGLTexture> m_texture;
0044
0045     EGLImage m_image = nullptr;
0046     bool m_needsRecreateTexture = false;
0047     bool m_allowDmaBuf = true;
0048
0049     struct {
0050         QImage texture;
0051         std::optional<QPoint> position;
0052         QPoint hotspot;
0053         bool dirty = false;
0054     } m_cursor;
0055     std::optional<QRegion> m_damage;
0056 };
0057
0058 class DiscardEglPixmapRunnable : public QRunnable
0059 {
0060 public:
0061     DiscardEglPixmapRunnable(EGLImageKHR image, QOpenGLTexture *texture)
0062         : m_image(image)
0063         , m_texture(texture)
0064     {
0065     }
0066
0067     void run() override
0068     {
0069         if (m_image != EGL_NO_IMAGE_KHR) {
0070             eglDestroyImageKHR(eglGetCurrentDisplay(), m_image);
0071         }
0072
0073         delete m_texture;
0074     }
0075
0076 private:
0077     const EGLImageKHR m_image;
0078     QOpenGLTexture *m_texture;
0079 };
0080
0081 PipeWireSourceItem::PipeWireSourceItem(QQuickItem *parent)
0082     : QQuickItem(parent)
0083     , d(new PipeWireSourceItemPrivate)
0084 {
0085     setFlag(ItemHasContents, true);
0086
0087     connect(this, &QQuickItem::visibleChanged, this, [this]() {
0088         setEnabled(isVisible());
0089         if (d->m_stream)
0090             d->m_stream->setActive(isVisible());
0091     });
0092 }
0093
0094 PipeWireSourceItem::~PipeWireSourceItem()
0095 {
0096     if (d->m_fd) {
0097         close(*d->m_fd);
0098     }
0099 }
0100
0101 void PipeWireSourceItem::itemChange(QQuickItem::ItemChange change, const QQuickItem::ItemChangeData &data)
0102 {
0103     switch (change) {
0104     case ItemVisibleHasChanged:
0105         setEnabled(isVisible());
0106         if (d->m_stream)
0107             d->m_stream->setActive(isVisible() && data.boolValue && isComponentComplete());
0108         break;
0109     case ItemSceneChange:
0110         d->m_needsRecreateTexture = true;
0111         releaseResources();
0112         break;
0113     default:
0114         break;
0115     }
0116
0117     QQuickItem::itemChange(change, data);
0118 }
0119
0120 void PipeWireSourceItem::releaseResources()
0121 {
0122     if (window() && (d->m_image || d->m_texture)) {
0123         window()->scheduleRenderJob(new DiscardEglPixmapRunnable(d->m_image, d->m_texture.release()), QQuickWindow::NoStage);
0124         d->m_image = EGL_NO_IMAGE_KHR;
0125     }
0126 }
0127
0128 void PipeWireSourceItem::setFd(uint fd)
0129 {
0130     if (fd == d->m_fd)
0131         return;
0132
0133     if (d->m_fd) {
0134         close(*d->m_fd);
0135     }
0136     d->m_fd = fd;
0137     refresh();
0138     Q_EMIT fdChanged(fd);
0139 }
0140
0141 void PipeWireSourceItem::resetFd()
0142 {
0143     if (!d->m_fd.has_value()) {
0144         return;
0145     }
0146
0147     setEnabled(false);
0148     close(*d->m_fd);
0149     d->m_fd.reset();
0150     d->m_stream.reset(nullptr);
0151     d->m_createNextTexture = [] {
0152         return nullptr;
0153     };
0154     Q_EMIT streamSizeChanged();
0155 }
0156
0157 void PipeWireSourceItem::refresh()
0158 {
0159     setEnabled(false);
0160
0161     if (!isComponentComplete()) {
0162         return;
0163     }
0164
0165     if (d->m_nodeId == 0) {
0166         releaseResources();
0167         d->m_stream.reset(nullptr);
0168         Q_EMIT streamSizeChanged();
0169
0170         d->m_createNextTexture = [] {
0171             return nullptr;
0172         };
0173     } else {
0174         d->m_stream.reset(new PipeWireSourceStream(this));
0175         d->m_stream->setAllowDmaBuf(d->m_allowDmaBuf);
0176         Q_EMIT streamSizeChanged();
0177         connect(d->m_stream.get(), &PipeWireSourceStream::streamParametersChanged, this, &PipeWireSourceItem::streamSizeChanged);
0178         connect(d->m_stream.get(), &PipeWireSourceStream::streamParametersChanged, this, &PipeWireSourceItem::usingDmaBufChanged);
0179
0180         d->m_stream->createStream(d->m_nodeId, d->m_fd.value_or(0));
0181         if (!d->m_stream->error().isEmpty()) {
0182             d->m_stream.reset(nullptr);
0183             d->m_nodeId = 0;
0184             return;
0185         }
0186         d->m_stream->setActive(isVisible() && isComponentComplete());
0187
0188         connect(d->m_stream.get(), &PipeWireSourceStream::frameReceived, this, &PipeWireSourceItem::processFrame);
0189         connect(d->m_stream.get(), &PipeWireSourceStream::stateChanged, this, &PipeWireSourceItem::stateChanged);
0190     }
0191     Q_EMIT stateChanged();
0192 }
0193
0194 void PipeWireSourceItem::setNodeId(uint nodeId)
0195 {
0196     if (nodeId == d->m_nodeId)
0197         return;
0198
0199     d->m_nodeId = nodeId;
0200     refresh();
0201     Q_EMIT nodeIdChanged(nodeId);
0202 }
0203
0204 class PipeWireRenderNode : public QSGNode
0205 {
0206 public:
0207     QSGImageNode *screenNode(QQuickWindow *window)
0208     {
0209         if (!m_screenNode) {
0210             m_screenNode = window->createImageNode();
0211             appendChildNode(m_screenNode);
0212         }
0213         return m_screenNode;
0214     }
0215     QSGImageNode *cursorNode(QQuickWindow *window)
0216     {
0217         if (!m_cursorNode) {
0218             m_cursorNode = window->createImageNode();
0219             appendChildNode(m_cursorNode);
0220         }
0221         return m_cursorNode;
0222     }
0223
0224     QSGImageNode *damageNode(QQuickWindow *window)
0225     {
0226         if (!m_damageNode) {
0227             m_damageNode = window->createImageNode();
0228             appendChildNode(m_damageNode);
0229         }
0230         return m_damageNode;
0231     }
0232
0233     void discardCursor()
0234     {
0235         if (m_cursorNode) {
0236             removeChildNode(m_cursorNode);
0237             delete m_cursorNode;
0238             m_cursorNode = nullptr;
0239         }
0240     }
0241
0242     void discardDamage()
0243     {
0244         if (m_damageNode) {
0245             removeChildNode(m_damageNode);
0246             delete m_damageNode;
0247             m_damageNode = nullptr;
0248         }
0249     }
0250
0251 private:
0252     QSGImageNode *m_screenNode = nullptr;
0253     QSGImageNode *m_cursorNode = nullptr;
0254     QSGImageNode *m_damageNode = nullptr;
0255 };
0256
0257 QSGNode *PipeWireSourceItem::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNodeData *)
0258 {
0259     if (Q_UNLIKELY(!d->m_createNextTexture)) {
0260         return node;
0261     }
0262
0263     auto texture = d->m_createNextTexture();
0264     if (!texture) {
0265         delete node;
0266         return nullptr;
0267     }
0268
0269     QSGImageNode *screenNode;
0270     auto pwNode = dynamic_cast<PipeWireRenderNode *>(node);
0271     if (!pwNode) {
0272         delete node;
0273         pwNode = new PipeWireRenderNode;
0274         screenNode = window()->createImageNode();
0275         screenNode->setOwnsTexture(true);
0276         pwNode->appendChildNode(screenNode);
0277     } else {
0278         screenNode = static_cast<QSGImageNode *>(pwNode->childAtIndex(0));
0279     }
0280     screenNode->setTexture(texture);
0281
0282     const auto br = boundingRect().toRect();
0283     QRect rect({0, 0}, texture->textureSize().scaled(br.size(), Qt::KeepAspectRatio));
0284     rect.moveCenter(br.center());
0285     screenNode->setRect(rect);
0286
0287     if (!d->m_cursor.position.has_value() || d->m_cursor.texture.isNull()) {
0288         pwNode->discardCursor();
0289     } else {
0290         QSGImageNode *cursorNode = pwNode->cursorNode(window());
0291         if (d->m_cursor.dirty || !cursorNode->texture()) {
0292             cursorNode->setTexture(window()->createTextureFromImage(d->m_cursor.texture));
0293             d->m_cursor.dirty = false;
0294         }
0295         const qreal scale = qreal(rect.width()) / texture->textureSize().width();
0296         cursorNode->setRect(QRectF{rect.topLeft() + (d->m_cursor.position.value() * scale), d->m_cursor.texture.size() * scale});
0297         Q_ASSERT(cursorNode->texture());
0298     }
0299
0300     if (!d->m_damage || d->m_damage->isEmpty()) {
0301         pwNode->discardDamage();
0302     } else {
0303         auto *damageNode = pwNode->damageNode(window());
0304         QImage damageImage(texture->textureSize(), QImage::Format_RGBA64_Premultiplied);
0305         damageImage.fill(Qt::transparent);
0306         QPainter p(&damageImage);
0307         p.setBrush(Qt::red);
0308         for (auto rect : *d->m_damage) {
0309             p.drawRect(rect);
0310         }
0311         damageNode->setTexture(window()->createTextureFromImage(damageImage));
0312         damageNode->setRect(rect);
0313         Q_ASSERT(damageNode->texture());
0314     }
0315     return pwNode;
0316 }
0317
0318 QString PipeWireSourceItem::error() const
0319 {
0320     return d->m_stream->error();
0321 }
0322
0323 void PipeWireSourceItem::processFrame(const PipeWireFrame &frame)
0324 {
0325     d->m_damage = frame.damage;
0326
0327     if (frame.cursor) {
0328         d->m_cursor.position = frame.cursor->position;
0329         d->m_cursor.hotspot = frame.cursor->hotspot;
0330         if (!frame.cursor->texture.isNull()) {
0331             d->m_cursor.dirty = true;
0332             d->m_cursor.texture = frame.cursor->texture;
0333         }
0334     } else {
0335         d->m_cursor.position = std::nullopt;
0336         d->m_cursor.hotspot = {};
0337     }
0338
0339     if (frame.dmabuf) {
0340         updateTextureDmaBuf(*frame.dmabuf, frame.format);
0341     } else if (frame.dataFrame) {
0342         updateTextureImage(frame.dataFrame);
0343     }
0344
0345     if (window() && window()->isVisible()) {
0346         update();
0347     }
0348 }
0349
0350 void PipeWireSourceItem::updateTextureDmaBuf(const DmaBufAttributes &attribs, spa_video_format format)
0351 {
0352     if (!window()) {
0353         qCWarning(PIPEWIRE_LOGGING) << "Window not available" << this;
0354         return;
0355     }
0356
0357     const auto openglContext = static_cast<QOpenGLContext *>(window()->rendererInterface()->getResource(window(), QSGRendererInterface::OpenGLContextResource));
0358     if (!openglContext || !d->m_stream) {
0359         qCWarning(PIPEWIRE_LOGGING) << "need a window and a context" << window();
0360         return;
0361     }
0362
0363     d->m_createNextTexture = [this, format, attribs]() -> QSGTexture * {
0364         const EGLDisplay display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay"));
0365         if (d->m_image) {
0366             eglDestroyImageKHR(display, d->m_image);
0367         }
0368         const auto size = d->m_stream->size();
0369         d->m_image = GLHelpers::createImage(display, attribs, PipeWireSourceStream::spaVideoFormatToDrmFormat(format), size, nullptr);
0370         if (d->m_image == EGL_NO_IMAGE_KHR) {
0371             d->m_stream->renegotiateModifierFailed(format, attribs.modifier);
0372             return nullptr;
0373         }
0374         if (!d->m_texture) {
0375             d->m_texture.reset(new QOpenGLTexture(QOpenGLTexture::Target2D));
0376             bool created = d->m_texture->create();
0377             Q_ASSERT(created);
0378         }
0379
0380         GLHelpers::initDebugOutput();
0381         d->m_texture->bind();
0382
0383         glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)d->m_image);
0384
0385         d->m_texture->setWrapMode(QOpenGLTexture::ClampToEdge);
0386         d->m_texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear);
0387         d->m_texture->release();
0388         d->m_texture->setSize(size.width(), size.height());
0389
0390         int textureId = d->m_texture->textureId();
0391         QQuickWindow::CreateTextureOption textureOption =
0392             format == SPA_VIDEO_FORMAT_ARGB || format == SPA_VIDEO_FORMAT_BGRA ? QQuickWindow::TextureHasAlphaChannel : QQuickWindow::TextureIsOpaque;
0393         setEnabled(true);
0394         return QNativeInterface::QSGOpenGLTexture::fromNative(textureId, window(), size, textureOption);
0395     };
0396 }
0397
0398 void PipeWireSourceItem::updateTextureImage(const std::shared_ptr<PipeWireFrameData> &data)
0399 {
0400     if (!window()) {
0401         qCWarning(PIPEWIRE_LOGGING) << "pass";
0402         return;
0403     }
0404
0405     d->m_createNextTexture = [this, data] {
0406         setEnabled(true);
0407         return window()->createTextureFromImage(data->toImage(), QQuickWindow::TextureIsOpaque);
0408     };
0409 }
0410
0411 void PipeWireSourceItem::componentComplete()
0412 {
0413     QQuickItem::componentComplete();
0414     if (d->m_nodeId != 0) {
0415         refresh();
0416     }
0417 }
0418
0419 PipeWireSourceItem::StreamState PipeWireSourceItem::state() const
0420 {
0421     if (!d->m_stream) {
0422         return StreamState::Unconnected;
0423     }
0424     switch (d->m_stream->state()) {
0425     case PW_STREAM_STATE_ERROR:
0426         return StreamState::Error;
0427     case PW_STREAM_STATE_UNCONNECTED:
0428         return StreamState::Unconnected;
0429     case PW_STREAM_STATE_CONNECTING:
0430         return StreamState::Connecting;
0431     case PW_STREAM_STATE_PAUSED:
0432         return StreamState::Paused;
0433     case PW_STREAM_STATE_STREAMING:
0434         return StreamState::Streaming;
0435     default:
0436         return StreamState::Error;
0437     }
0438 }
0439
0440 uint PipeWireSourceItem::fd() const
0441 {
0442     return d->m_fd.value_or(0);
0443 }
0444
0445 uint PipeWireSourceItem::nodeId() const
0446 {
0447     return d->m_nodeId;
0448 }
0449
0450 QSize PipeWireSourceItem::streamSize() const
0451 {
0452     if (!d->m_stream) {
0453         return QSize();
0454     }
0455     return d->m_stream->size();
0456 }
0457
0458 bool PipeWireSourceItem::usingDmaBuf() const
0459 {
0460     return d->m_stream && d->m_stream->usingDmaBuf();
0461 }
0462
0463 bool PipeWireSourceItem::allowDmaBuf() const
0464 {
0465     return d->m_stream && d->m_stream->allowDmaBuf();
0466 }
0467
0468 void PipeWireSourceItem::setAllowDmaBuf(bool allowed)
0469 {
0470     d->m_allowDmaBuf = allowed;
0471     if (d->m_stream) {
0472         d->m_stream->setAllowDmaBuf(allowed);
0473     }
0474 }
0475
0476 #include "moc_pipewiresourceitem.cpp"